thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Limit backtracking in parser

Browse Source
This commit is contained in:
Alexey Milovidov 2024-03-17 19:53:58 +01:00
parent bfea36877b
commit 01136bbc3b
138 changed files with 549 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
programs/compressor/Compressor.cpp
View File

@ -143,7 +143,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
ParserCodec codec_parser;
std::string codecs_line = boost::algorithm::join(codecs, ",");
auto ast = parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
auto ast = parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
codec = CompressionCodecFactory::instance().get(ast, nullptr);
}
else

2
programs/format/Format.cpp
View File

@ -234,7 +234,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
size_t approx_query_length = multiple ? find_first_symbols<';'>(pos, end) - pos : end - pos;
ASTPtr res = parseQueryAndMovePosition(
parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth);
parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth, cmd_settings.max_parser_backtracks);
std::unique_ptr<ReadBuffer> insert_query_payload = nullptr;
/// If the query is INSERT ... VALUES, then we will try to parse the data.

3
programs/keeper-client/KeeperClient.cpp
View File

@ -44,7 +44,7 @@ String KeeperClient::executeFourLetterCommand(const String & command)
std::vector<String> KeeperClient::getCompletions(const String & prefix) const
{
Tokens tokens(prefix.data(), prefix.data() + prefix.size(), 0, false);
IParser::Pos pos(tokens, 0);
IParser::Pos pos(tokens, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
if (pos->type != TokenType::BareWord)
return registered_commands_and_four_letter_words;
@ -278,6 +278,7 @@ bool KeeperClient::processQueryText(const String & text)
/* allow_multi_statements = */ true,
/* max_query_size = */ 0,
/* max_parser_depth = */ 0,
/* max_parser_backtracks = */ 0,
/* skip_insignificant = */ false);
if (!res)

2
src/Access/AccessEntityIO.cpp
View File

@ -62,7 +62,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition)
const char * end = begin + definition.size();
while (pos < end)
{
queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH));
queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS));
while (isWhitespaceASCII(*pos) || *pos == ';')
++pos;
}

2
src/Access/RowPolicyCache.cpp
View File

@ -86,7 +86,7 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_)
try
{
ParserExpression parser;
parsed_filters[filter_type_i] = parseQuery(parser, filter, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
parsed_filters[filter_type_i] = parseQuery(parser, filter, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
}
catch (...)
{

2
src/Access/UsersConfigAccessStorage.cpp
View File

@ -66,7 +66,7 @@ namespace
String error_message;
const char * pos = string_query.data();
auto ast = tryParseQuery(parser, pos, pos + string_query.size(), error_message, false, "", false, 0, 0);
auto ast = tryParseQuery(parser, pos, pos + string_query.size(), error_message, false, "", false, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true);
if (!ast)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to parse grant query. Error: {}", error_message);

3
src/AggregateFunctions/parseAggregateFunctionParameters.cpp
View File

@ -81,7 +81,8 @@ void getAggregateFunctionNameAndParametersArray(
ParserExpressionList params_parser(false);
ASTPtr args_ast = parseQuery(params_parser,
parameters_str.data(), parameters_str.data() + parameters_str.size(),
"parameters of aggregate function in " + error_context, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
"parameters of aggregate function in " + error_context,
0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
if (args_ast->children.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Incorrect list of parameters to aggregate function {}",

2
src/Backups/BackupInfo.cpp
View File

@ -25,7 +25,7 @@ String BackupInfo::toString() const
BackupInfo BackupInfo::fromString(const String & str)
{
ParserIdentifierWithOptionalParameters parser;
ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
return fromAST(*ast);
}

4
src/Backups/RestorerFromBackup.cpp
View File

@ -424,7 +424,7 @@ void RestorerFromBackup::findTableInBackupImpl(const QualifiedTableName & table_
readStringUntilEOF(create_query_str, *read_buffer);
read_buffer.reset();
ParserCreateQuery create_parser;
ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
applyCustomStoragePolicy(create_table_query);
renameDatabaseAndTableNameInCreateQuery(create_table_query, renaming_map, context->getGlobalContext());
String create_table_query_str = serializeAST(*create_table_query);
@ -534,7 +534,7 @@ void RestorerFromBackup::findDatabaseInBackupImpl(const String & database_name_i
readStringUntilEOF(create_query_str, *read_buffer);
read_buffer.reset();
ParserCreateQuery create_parser;
ASTPtr create_database_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr create_database_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
renameDatabaseAndTableNameInCreateQuery(create_database_query, renaming_map, context->getGlobalContext());
String create_database_query_str = serializeAST(*create_database_query);

24
src/Client/ClientBase.cpp
View File

@ -345,7 +345,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
if (dialect == Dialect::kusto)
parser = std::make_unique<ParserKQLStatement>(end, global_context->getSettings().allow_settings_after_format_in_insert);
else if (dialect == Dialect::prql)
parser = std::make_unique<ParserPRQLQuery>(max_length, settings.max_parser_depth);
parser = std::make_unique<ParserPRQLQuery>(max_length, settings.max_parser_depth, settings.max_parser_backtracks);
else
parser = std::make_unique<ParserQuery>(end, global_context->getSettings().allow_settings_after_format_in_insert);
@ -353,9 +353,9 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
{
String message;
if (dialect == Dialect::kusto)
res = tryParseKQLQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
res = tryParseKQLQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks, true);
else
res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth);
res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks, true);
if (!res)
{
@ -366,9 +366,9 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
else
{
if (dialect == Dialect::kusto)
res = parseKQLQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
res = parseKQLQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks);
else
res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth);
res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks);
}
if (is_interactive)
@ -385,12 +385,12 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
/// Consumes trailing semicolons and tries to consume the same-line trailing comment.
void ClientBase::adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth)
void ClientBase::adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth, uint32_t max_parser_backtracks)
{
// We have to skip the trailing semicolon that might be left
// after VALUES parsing or just after a normal semicolon-terminated query.
Tokens after_query_tokens(this_query_end, all_queries_end);
IParser::Pos after_query_iterator(after_query_tokens, max_parser_depth);
IParser::Pos after_query_iterator(after_query_tokens, max_parser_depth, max_parser_backtracks);
while (after_query_iterator.isValid() && after_query_iterator->type == TokenType::Semicolon)
{
this_query_end = after_query_iterator->end;
@ -1984,6 +1984,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
return MultiQueryProcessingStage::QUERIES_END;
unsigned max_parser_depth = static_cast<unsigned>(global_context->getSettingsRef().max_parser_depth);
unsigned max_parser_backtracks = static_cast<unsigned>(global_context->getSettingsRef().max_parser_backtracks);
// If there are only comments left until the end of file, we just
// stop. The parser can't handle this situation because it always
@ -1994,7 +1995,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
// and it makes more sense to treat them as such.
{
Tokens tokens(this_query_begin, all_queries_end);
IParser::Pos token_iterator(tokens, max_parser_depth);
IParser::Pos token_iterator(tokens, max_parser_depth, max_parser_backtracks);
if (!token_iterator.isValid())
return MultiQueryProcessingStage::QUERIES_END;
}
@ -2015,7 +2016,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
if (ignore_error)
{
Tokens tokens(this_query_begin, all_queries_end);
IParser::Pos token_iterator(tokens, max_parser_depth);
IParser::Pos token_iterator(tokens, max_parser_depth, max_parser_backtracks);
while (token_iterator->type != TokenType::Semicolon && token_iterator.isValid())
++token_iterator;
this_query_begin = token_iterator->end;
@ -2055,7 +2056,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
// after we have processed the query. But even this guess is
// beneficial so that we see proper trailing comments in "echo" and
// server log.
adjustQueryEnd(this_query_end, all_queries_end, max_parser_depth);
adjustQueryEnd(this_query_end, all_queries_end, max_parser_depth, max_parser_backtracks);
return MultiQueryProcessingStage::EXECUTE_QUERY;
}
@ -2251,7 +2252,8 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
this_query_end = insert_ast->end;
adjustQueryEnd(
this_query_end, all_queries_end,
static_cast<unsigned>(global_context->getSettingsRef().max_parser_depth));
static_cast<unsigned>(global_context->getSettingsRef().max_parser_depth),
static_cast<unsigned>(global_context->getSettingsRef().max_parser_backtracks));
}
// Report error.

2
src/Client/ClientBase.h
View File

@ -94,7 +94,7 @@ protected:
void processParsedSingleQuery(const String & full_query, const String & query_to_execute,
ASTPtr parsed_query, std::optional<bool> echo_query_ = {}, bool report_error = false);
static void adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth);
static void adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth, uint32_t max_parser_backtracks);
ASTPtr parseQuery(const char *& pos, const char * end, bool allow_multi_statements) const;
static void setupSignalHandler();

6
src/Client/QueryFuzzer.cpp
View File

@ -569,7 +569,8 @@ void QueryFuzzer::fuzzColumnDeclaration(ASTColumnDeclaration & column)
auto data_type = fuzzDataType(DataTypeFactory::instance().get(column.type));
ParserDataType parser;
column.type = parseQuery(parser, data_type->getName(), DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH);
column.type = parseQuery(parser, data_type->getName(),
DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
}
}
@ -821,7 +822,8 @@ static ASTPtr tryParseInsertQuery(const String & full_query)
ParserInsertQuery parser(end, false);
String message;
return tryParseQuery(parser, pos, end, message, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH);
return tryParseQuery(parser, pos, end, message, false, "", false,
DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true);
}
ASTs QueryFuzzer::getInsertQueriesForFuzzedTables(const String & full_query)

2
src/Common/NamedCollections/NamedCollectionUtils.cpp
View File

@ -302,7 +302,7 @@ private:
readStringUntilEOF(query, in);
ParserCreateNamedCollectionQuery parser;
auto ast = parseQuery(parser, query, "in file " + path, 0, settings.max_parser_depth);
auto ast = parseQuery(parser, query, "in file " + path, 0, settings.max_parser_depth, settings.max_parser_backtracks);
const auto & create_query = ast->as<const ASTCreateNamedCollectionQuery &>();
return create_query;
}

2
src/Compression/tests/gtest_compressionCodec.cpp
View File

@ -442,7 +442,7 @@ CompressionCodecPtr makeCodec(const std::string & codec_string, const DataTypePt
{
const std::string codec_statement = "(" + codec_string + ")";
Tokens tokens(codec_statement.begin().base(), codec_statement.end().base());
IParser::Pos token_iterator(tokens, 0);
IParser::Pos token_iterator(tokens, 0, 0);
Expected expected;
ASTPtr codec_ast;

2
src/Core/Defines.h
View File

@ -63,6 +63,8 @@ static constexpr auto DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC = 120;
/// Default limit on recursion depth of recursive descend parser.
static constexpr auto DBMS_DEFAULT_MAX_PARSER_DEPTH = 1000;
/// Default limit on the amount of backtracking of recursive descend parser.
static constexpr auto DBMS_DEFAULT_MAX_PARSER_BACKTRACKS = 1000000;
/// Default limit on query size.
static constexpr auto DBMS_DEFAULT_MAX_QUERY_SIZE = 262144;

1
src/Core/Settings.h
View File

@ -607,6 +607,7 @@ class IColumn;
M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \
M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \
M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \
M(UInt64, max_parser_backtracks, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, "Maximum parser backtracking (how many times it tries different alternatives in the recursive descend parsing process).", 0) \
M(Bool, allow_settings_after_format_in_insert, false, "Allow SETTINGS after FORMAT, but note, that this is not always safe (note: this is a compatibility setting).", 0) \
M(Seconds, periodic_live_view_refresh, 60, "Interval after which periodically refreshed live view is forced to refresh.", 0) \
M(Bool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \

1
src/Core/SettingsChangesHistory.h
View File

@ -95,6 +95,7 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
{"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."},
{"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
{"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
{"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
}},
{"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
{"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},

5
src/DataTypes/DataTypeFactory.cpp
View File

@ -56,13 +56,14 @@ DataTypePtr DataTypeFactory::getImpl(const String & full_name) const
{
String out_err;
const char * start = full_name.data();
ast = tryParseQuery(parser, start, start + full_name.size(), out_err, false, "data type", false, DBMS_DEFAULT_MAX_QUERY_SIZE, data_type_max_parse_depth);
ast = tryParseQuery(parser, start, start + full_name.size(), out_err, false, "data type", false,
DBMS_DEFAULT_MAX_QUERY_SIZE, data_type_max_parse_depth, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true);
if (!ast)
return nullptr;
}
else
{
ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", false, data_type_max_parse_depth);
ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", false, data_type_max_parse_depth, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
}
return getImpl<nullptr_on_error>(ast);

3
src/Databases/DDLDependencyVisitor.cpp
View File

@ -444,8 +444,9 @@ namespace
ParserSelectWithUnionQuery parser;
String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name);
String fixed_query = removeWhereConditionPlaceholder(query);
const Settings & settings = data.context->getSettingsRef();
ASTPtr select = parseQuery(parser, fixed_query, description,
data.context->getSettingsRef().max_query_size, data.context->getSettingsRef().max_parser_depth);
settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
DDLDependencyVisitor::Visitor visitor{data};
visitor.visit(select);

4
src/Databases/DatabaseDictionary.cpp
View File

@ -115,7 +115,7 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message,
/* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth);
/* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth, settings.max_parser_backtracks, true);
if (!ast && throw_on_error)
throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR);
@ -134,7 +134,7 @@ ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const
}
auto settings = getContext()->getSettingsRef();
ParserCreateQuery parser;
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
}
void DatabaseDictionary::shutdown()

2
src/Databases/DatabaseFilesystem.cpp
View File

@ -187,7 +187,7 @@ ASTPtr DatabaseFilesystem::getCreateDatabaseQuery() const
const String query = fmt::format("CREATE DATABASE {} ENGINE = Filesystem('{}')", backQuoteIfNeed(getDatabaseName()), path);
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
{

2
src/Databases/DatabaseHDFS.cpp
View File

@ -183,7 +183,7 @@ ASTPtr DatabaseHDFS::getCreateDatabaseQuery() const
ParserCreateQuery parser;
const String query = fmt::format("CREATE DATABASE {} ENGINE = HDFS('{}')", backQuoteIfNeed(getDatabaseName()), source);
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
{

18
src/Databases/DatabaseOnDisk.cpp
View File

@ -526,7 +526,7 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
/// If database.sql doesn't exist, then engine is Ordinary
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
}
if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
@ -707,7 +707,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + metadata_file_path, /* allow_multi_statements = */ false, 0, settings.max_parser_depth);
"in file " + metadata_file_path, /* allow_multi_statements = */ false, 0, settings.max_parser_depth, settings.max_parser_backtracks, true);
if (!ast && throw_on_error)
throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR);
@ -765,12 +765,14 @@ ASTPtr DatabaseOnDisk::getCreateQueryFromStorage(const String & table_name, cons
auto ast_storage = std::make_shared<ASTStorage>();
ast_storage->set(ast_storage->engine, ast_engine);
unsigned max_parser_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
auto create_table_query = DB::getCreateQueryFromStorage(storage,
ast_storage,
false,
max_parser_depth,
throw_on_error);
const Settings & settings = getContext()->getSettingsRef();
auto create_table_query = DB::getCreateQueryFromStorage(
storage,
ast_storage,
false,
static_cast<unsigned>(settings.max_parser_depth),
static_cast<unsigned>(settings.max_parser_backtracks),
throw_on_error);
create_table_query->set(create_table_query->as<ASTCreateQuery>()->comment,
std::make_shared<ASTLiteral>("SYSTEM TABLE is built on the fly."));

2
src/Databases/DatabaseOrdinary.cpp
View File

@ -469,7 +469,7 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta
statement.data() + statement.size(),
"in file " + table_metadata_path,
0,
local_context->getSettingsRef().max_parser_depth);
local_context->getSettingsRef().max_parser_depth, local_context->getSettingsRef().max_parser_backtracks);
applyMetadataChangesToCreateQuery(ast, metadata);

7
src/Databases/DatabaseReplicated.cpp
View File

@ -812,7 +812,8 @@ static UUID getTableUUIDIfReplicated(const String & metadata, ContextPtr context
ParserCreateQuery parser;
auto size = context->getSettingsRef().max_query_size;
auto depth = context->getSettingsRef().max_parser_depth;
ASTPtr query = parseQuery(parser, metadata, size, depth);
auto backtracks = context->getSettingsRef().max_parser_backtracks;
ASTPtr query = parseQuery(parser, metadata, size, depth, backtracks);
const ASTCreateQuery & create = query->as<const ASTCreateQuery &>();
if (!create.storage || !create.storage->engine)
return UUIDHelpers::Nil;
@ -1234,7 +1235,7 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node
{
ParserCreateQuery parser;
String description = "in ZooKeeper " + zookeeper_path + "/metadata/" + node_name;
auto ast = parseQuery(parser, query, description, 0, getContext()->getSettingsRef().max_parser_depth);
auto ast = parseQuery(parser, query, description, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks);
auto & create = ast->as<ASTCreateQuery &>();
if (create.uuid == UUIDHelpers::Nil || create.getTable() != TABLE_WITH_UUID_NAME_PLACEHOLDER || create.database)
@ -1559,7 +1560,7 @@ DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, cons
for (const auto & [table_name, metadata] : snapshot)
{
ParserCreateQuery parser;
auto create_table_query = parseQuery(parser, metadata, 0, getContext()->getSettingsRef().max_parser_depth);
auto create_table_query = parseQuery(parser, metadata, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks);
auto & create = create_table_query->as<ASTCreateQuery &>();
create.attach = false;

2
src/Databases/DatabaseS3.cpp
View File

@ -191,7 +191,7 @@ ASTPtr DatabaseS3::getCreateDatabaseQuery() const
creation_args += fmt::format(", '{}', '{}'", config.access_key_id.value(), config.secret_access_key.value());
const String query = fmt::format("CREATE DATABASE {} ENGINE = S3({})", backQuoteIfNeed(getDatabaseName()), creation_args);
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
{

5
src/Databases/DatabasesCommon.cpp
View File

@ -108,7 +108,8 @@ void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemo
}
ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_storage, bool only_ordinary, uint32_t max_parser_depth, bool throw_on_error)
ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_storage, bool only_ordinary,
uint32_t max_parser_depth, uint32_t max_parser_backtracks, bool throw_on_error)
{
auto table_id = storage->getStorageID();
auto metadata_ptr = storage->getInMemoryMetadataPtr();
@ -148,7 +149,7 @@ ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_
Expected expected;
expected.max_parsed_pos = string_end;
Tokens tokens(type_name.c_str(), string_end);
IParser::Pos pos(tokens, max_parser_depth);
IParser::Pos pos(tokens, max_parser_depth, max_parser_backtracks);
ParserDataType parser;
if (!parser.parse(pos, ast_type, expected))
{

3
src/Databases/DatabasesCommon.h
View File

@ -13,7 +13,8 @@ namespace DB
{
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata);
ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_storage, bool only_ordinary, uint32_t max_parser_depth, bool throw_on_error);
ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_storage, bool only_ordinary,
uint32_t max_parser_depth, uint32_t max_parser_backtracks, bool throw_on_error);
/// Cleans a CREATE QUERY from temporary flags like "IF NOT EXISTS", "OR REPLACE", "AS SELECT" (for non-views), etc.
void cleanupObjectDefinitionFromTemporaryFlags(ASTCreateQuery & query);

14
src/Databases/MySQL/DatabaseMySQL.cpp
View File

@ -174,12 +174,14 @@ ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, Context
ast_storage->settings = nullptr;
}
unsigned max_parser_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
auto create_table_query = DB::getCreateQueryFromStorage(storage,
table_storage_define,
true,
max_parser_depth,
throw_on_error);
const Settings & settings = getContext()->getSettingsRef();
auto create_table_query = DB::getCreateQueryFromStorage(
storage,
table_storage_define,
true,
static_cast<unsigned>(settings.max_parser_depth),
static_cast<unsigned>(settings.max_parser_backtracks),
throw_on_error);
return create_table_query;
}

2
src/Databases/MySQL/tryConvertStringLiterals.cpp
View File

@ -61,7 +61,7 @@ static bool tryReadCharset(
bool tryConvertStringLiterals(String & query)
{
Tokens tokens(query.data(), query.data() + query.size());
IParser::Pos pos(tokens, 0);
IParser::Pos pos(tokens, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
Expected expected;
String rewritten_query;
rewritten_query.reserve(query.size());

2
src/Databases/MySQL/tryParseTableIDFromDDL.cpp
View File

@ -10,7 +10,7 @@ StorageID tryParseTableIDFromDDL(const String & query, const String & default_da
{
bool is_ddl = false;
Tokens tokens(query.data(), query.data() + query.size());
IParser::Pos pos(tokens, 0);
IParser::Pos pos(tokens, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
Expected expected;
if (ParserKeyword("CREATE TEMPORARY TABLE").ignore(pos, expected) || ParserKeyword("CREATE TABLE").ignore(pos, expected))
{

2
src/Databases/MySQL/tryQuoteUnrecognizedTokens.cpp
View File

@ -37,7 +37,7 @@ static void quoteLiteral(
bool tryQuoteUnrecognizedTokens(String & query)
{
Tokens tokens(query.data(), query.data() + query.size());
IParser::Pos pos(tokens, 0);
IParser::Pos pos(tokens, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
Expected expected;
String rewritten_query;
const char * copy_from = query.data();

6
src/Databases/SQLite/DatabaseSQLite.cpp
View File

@ -194,10 +194,10 @@ ASTPtr DatabaseSQLite::getCreateTableQueryImpl(const String & table_name, Contex
/// Add table_name to engine arguments
storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 1, std::make_shared<ASTLiteral>(table_id.table_name));
unsigned max_parser_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
const Settings & settings = getContext()->getSettingsRef();
auto create_table_query = DB::getCreateQueryFromStorage(storage, table_storage_define, true,
max_parser_depth,
throw_on_error);
static_cast<uint32_t>(settings.max_parser_depth), static_cast<uint32_t>(settings.max_parser_backtracks), throw_on_error);
return create_table_query;
}

8
src/Dictionaries/tests/gtest_dictionary_configuration.cpp
View File

@ -48,7 +48,7 @@ TEST(ConvertDictionaryAST, SimpleDictConfiguration)
" COMMENT 'hello world!'";
ParserCreateDictionaryQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTCreateQuery * create = ast->as<ASTCreateQuery>();
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, getContext().context);
@ -119,7 +119,7 @@ TEST(ConvertDictionaryAST, TrickyAttributes)
" SOURCE(CLICKHOUSE(HOST 'localhost'))";
ParserCreateDictionaryQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTCreateQuery * create = ast->as<ASTCreateQuery>();
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, getContext().context);
@ -164,7 +164,7 @@ TEST(ConvertDictionaryAST, ComplexKeyAndLayoutWithParams)
" LIFETIME(MIN 1 MAX 10)";
ParserCreateDictionaryQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTCreateQuery * create = ast->as<ASTCreateQuery>();
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, getContext().context);
@ -215,7 +215,7 @@ TEST(ConvertDictionaryAST, ComplexSource)
" RANGE(MIN second_column MAX third_column)";
ParserCreateDictionaryQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTCreateQuery * create = ast->as<ASTCreateQuery>();
DictionaryConfigurationPtr config = getDictionaryConfigurationFromAST(*create, getContext().context);
/// source

2
src/Formats/SchemaInferenceUtils.cpp
View File

@ -1054,7 +1054,7 @@ namespace
{
if (depth > settings.max_parser_depth)
throw Exception(ErrorCodes::TOO_DEEP_RECURSION,
"Maximum parse depth ({}) exceeded. Consider raising max_parser_depth setting.", settings.max_parser_depth);
"Maximum parse depth ({}) exceeded. Consider raising max_parser_depth setting.", settings.max_parser_depth);
assertChar('{', buf);
skipWhitespaceIfAny(buf);

9
src/Functions/FunctionSQLJSON.h
View File

@ -123,7 +123,7 @@ public:
class Executor
{
public:
static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth, const ContextPtr & context)
static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth, uint32_t parse_backtracks, const ContextPtr & context)
{
MutableColumnPtr to{result_type->createColumn()};
to->reserve(input_rows_count);
@ -161,7 +161,7 @@ public:
/// Tokenize the query
Tokens tokens(query.data(), query.data() + query.size());
/// Max depth 0 indicates that depth is not limited
IParser::Pos token_iterator(tokens, parse_depth);
IParser::Pos token_iterator(tokens, parse_depth, parse_backtracks);
/// Parse query and create AST tree
Expected expected;
@ -232,16 +232,17 @@ public:
/// 3. Parser(Tokens, ASTPtr) -> complete AST
/// 4. Execute functions: call getNextItem on generator and handle each item
unsigned parse_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
unsigned parse_backtracks = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_backtracks);
#if USE_SIMDJSON
if (getContext()->getSettingsRef().allow_simdjson)
return FunctionSQLJSONHelpers::Executor<
Name,
Impl<SimdJSONParser, JSONStringSerializer<SimdJSONParser::Element, SimdJSONElementFormatter>>,
SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, getContext());
SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, parse_backtracks, getContext());
#endif
return FunctionSQLJSONHelpers::
Executor<Name, Impl<DummyJSONParser, DefaultJSONStringSerializer<DummyJSONParser::Element>>, DummyJSONParser>::run(
arguments, result_type, input_rows_count, parse_depth, getContext());
arguments, result_type, input_rows_count, parse_depth, parse_backtracks, getContext());
}
};

2
src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp
View File

@ -128,7 +128,7 @@ restoreUserDefinedSQLObjects(RestorerFromBackup & restorer, const String & data_
statement_def.data() + statement_def.size(),
"in file " + filepath + " from backup " + backup->getNameForLogging(),
0,
context->getSettingsRef().max_parser_depth);
context->getSettingsRef().max_parser_depth, context->getSettingsRef().max_parser_backtracks);
break;
}
}

3
src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp
View File

@ -92,7 +92,8 @@ ASTPtr UserDefinedSQLObjectsDiskStorage::tryLoadObject(UserDefinedSQLObjectType
object_create_query.data() + object_create_query.size(),
"",
0,
global_context->getSettingsRef().max_parser_depth);
global_context->getSettingsRef().max_parser_depth,
global_context->getSettingsRef().max_parser_backtracks);
return ast;
}
}

3
src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp
View File

@ -314,7 +314,8 @@ ASTPtr UserDefinedSQLObjectsZooKeeperStorage::parseObjectData(const String & obj
object_data.data() + object_data.size(),
"",
0,
global_context->getSettingsRef().max_parser_depth);
global_context->getSettingsRef().max_parser_depth,
global_context->getSettingsRef().max_parser_backtracks);
return ast;
}
}

168
src/Functions/formatQuery.cpp
View File

@ -17,6 +17,9 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
}
namespace
{
enum class OutputFormatting
{
SingleLine,
@ -29,21 +32,16 @@ enum class ErrorHandling
Null
};
template <OutputFormatting output_formatting, ErrorHandling error_handling, typename Name>
class FunctionFormatQuery : public IFunction
{
public:
static constexpr auto name = Name::name;
static FunctionPtr create(ContextPtr context)
{
const auto & settings = context->getSettings();
return std::make_shared<FunctionFormatQuery>(settings.max_query_size, settings.max_parser_depth);
}
FunctionFormatQuery(size_t max_query_size_, size_t max_parser_depth_)
: max_query_size(max_query_size_)
, max_parser_depth(max_parser_depth_)
FunctionFormatQuery(ContextPtr context, String name_, OutputFormatting output_formatting_, ErrorHandling error_handling_)
: name(name_), output_formatting(output_formatting_), error_handling(error_handling_)
{
const Settings & settings = context->getSettings();
max_query_size = settings.max_query_size;
max_parser_depth = settings.max_parser_depth;
max_parser_backtracks = settings.max_parser_backtracks;
}
String getName() const override { return name; }
@ -59,7 +57,7 @@ public:
validateFunctionArgumentTypes(*this, arguments, args);
DataTypePtr string_type = std::make_shared<DataTypeString>();
if constexpr (error_handling == ErrorHandling::Null)
if (error_handling == ErrorHandling::Null)
return std::make_shared<DataTypeNullable>(string_type);
else
return string_type;
@ -70,7 +68,7 @@ public:
const ColumnPtr col_query = arguments[0].column;
ColumnUInt8::MutablePtr col_null_map;
if constexpr (error_handling == ErrorHandling::Null)
if (error_handling == ErrorHandling::Null)
col_null_map = ColumnUInt8::create(input_rows_count, 0);
if (const ColumnString * col_query_string = checkAndGetColumn<ColumnString>(col_query.get()))
@ -78,7 +76,7 @@ public:
auto col_res = ColumnString::create();
formatVector(col_query_string->getChars(), col_query_string->getOffsets(), col_res->getChars(), col_res->getOffsets(), col_null_map);
if constexpr (error_handling == ErrorHandling::Null)
if (error_handling == ErrorHandling::Null)
return ColumnNullable::create(std::move(col_res), std::move(col_null_map));
else
return col_res;
@ -113,11 +111,11 @@ private:
try
{
ast = parseQuery(parser, begin, end, /*query_description*/ {}, max_query_size, max_parser_depth);
ast = parseQuery(parser, begin, end, /*query_description*/ {}, max_query_size, max_parser_depth, max_parser_backtracks);
}
catch (...)
{
if constexpr (error_handling == ErrorHandling::Null)
if (error_handling == ErrorHandling::Null)
{
const size_t res_data_new_size = res_data_size + 1;
if (res_data_new_size > res_data.size())
@ -135,7 +133,6 @@ private:
}
else
{
static_assert(error_handling == ErrorHandling::Exception);
throw;
}
}
@ -160,92 +157,91 @@ private:
res_data.resize(res_data_size);
}
const size_t max_query_size;
const size_t max_parser_depth;
String name;
OutputFormatting output_formatting;
ErrorHandling error_handling;
size_t max_query_size;
size_t max_parser_depth;
size_t max_parser_backtracks;
};
struct NameFormatQuery
{
static constexpr auto name = "formatQuery";
};
struct NameFormatQueryOrNull
{
static constexpr auto name = "formatQueryOrNull";
};
struct NameFormatQuerySingleLine
{
static constexpr auto name = "formatQuerySingleLine";
};
struct NameFormatQuerySingleLineOrNull
{
static constexpr auto name = "formatQuerySingleLineOrNull";
};
}
REGISTER_FUNCTION(formatQuery)
{
factory.registerFunction<FunctionFormatQuery<OutputFormatting::MultiLine, ErrorHandling::Exception, NameFormatQuery>>(FunctionDocumentation{
.description = "Returns a formatted, possibly multi-line, version of the given SQL query. Throws in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuery(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuery('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT\n"
" a,\n"
" b\n"
"FROM tab\n"
"WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
factory.registerFunction(
"formatQuery",
[](ContextPtr context) { return std::make_shared<FunctionFormatQuery>(context, "formatQuery", OutputFormatting::MultiLine, ErrorHandling::Exception); },
FunctionDocumentation{
.description = "Returns a formatted, possibly multi-line, version of the given SQL query. Throws in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuery(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuery('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT\n"
" a,\n"
" b\n"
"FROM tab\n"
"WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
}
REGISTER_FUNCTION(formatQueryOrNull)
{
factory.registerFunction<FunctionFormatQuery<OutputFormatting::MultiLine, ErrorHandling::Null, NameFormatQueryOrNull>>(FunctionDocumentation{
.description = "Returns a formatted, possibly multi-line, version of the given SQL query. Returns NULL in case of a parsing error.\n[example:multiline]",
.syntax = "formatQueryOrNull(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuery('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT\n"
" a,\n"
" b\n"
"FROM tab\n"
"WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
factory.registerFunction(
"formatQueryOrNull",
[](ContextPtr context) { return std::make_shared<FunctionFormatQuery>(context, "formatQueryOrNull", OutputFormatting::MultiLine, ErrorHandling::Null); },
FunctionDocumentation{
.description = "Returns a formatted, possibly multi-line, version of the given SQL query. Returns NULL in case of a parsing error.\n[example:multiline]",
.syntax = "formatQueryOrNull(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuery('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT\n"
" a,\n"
" b\n"
"FROM tab\n"
"WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
}
REGISTER_FUNCTION(formatQuerySingleLine)
{
factory.registerFunction<FunctionFormatQuery<OutputFormatting::SingleLine, ErrorHandling::Exception, NameFormatQuerySingleLine>>(FunctionDocumentation{
.description = "Like formatQuery() but the returned formatted string contains no line breaks. Throws in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuerySingleLine(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuerySingleLine('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT a, b FROM tab WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
factory.registerFunction(
"formatQuerySingleLine",
[](ContextPtr context) { return std::make_shared<FunctionFormatQuery>(context, "formatQuerySingleLine", OutputFormatting::SingleLine, ErrorHandling::Exception); },
FunctionDocumentation{
.description = "Like formatQuery() but the returned formatted string contains no line breaks. Throws in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuerySingleLine(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuerySingleLine('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT a, b FROM tab WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
}
REGISTER_FUNCTION(formatQuerySingleLineOrNull)
{
factory.registerFunction<FunctionFormatQuery<OutputFormatting::SingleLine, ErrorHandling::Null, NameFormatQuerySingleLineOrNull>>(FunctionDocumentation{
.description = "Like formatQuery() but the returned formatted string contains no line breaks. Returns NULL in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuerySingleLineOrNull(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuerySingleLine('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT a, b FROM tab WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
factory.registerFunction(
"formatQuerySingleLineOrNull",
[](ContextPtr context) { return std::make_shared<FunctionFormatQuery>(context, "formatQuerySingleLineOrNull", OutputFormatting::SingleLine, ErrorHandling::Null); },
FunctionDocumentation{
.description = "Like formatQuery() but the returned formatted string contains no line breaks. Returns NULL in case of a parsing error.\n[example:multiline]",
.syntax = "formatQuerySingleLineOrNull(query)",
.arguments = {{"query", "The SQL query to be formatted. [String](../../sql-reference/data-types/string.md)"}},
.returned_value = "The formatted query. [String](../../sql-reference/data-types/string.md).",
.examples{
{"multiline",
"SELECT formatQuerySingleLine('select a, b FRom tab WHERE a > 3 and b < 3');",
"SELECT a, b FROM tab WHERE (a > 3) AND (b < 3)"}},
.categories{"Other"}});
}
}

2
src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp
View File

@ -206,7 +206,7 @@ namespace
if (!filter.empty())
{
ParserExpression parser;
ASTPtr expr = parseQuery(parser, filter, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr expr = parseQuery(parser, filter, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
query->filters.emplace_back(type, std::move(expr));
}
}

10
src/Interpreters/AsynchronousMetricLog.cpp
View File

@ -21,31 +21,31 @@ ColumnsDescription AsynchronousMetricLogElement::getColumnsDescription()
{
"hostname",
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()),
parseQuery(codec_parser, "(ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH),
parseQuery(codec_parser, "(ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS),
"Hostname of the server executing the query."
},
{
"event_date",
std::make_shared<DataTypeDate>(),
parseQuery(codec_parser, "(Delta(2), ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH),
parseQuery(codec_parser, "(Delta(2), ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS),
"Event date."
},
{
"event_time",
std::make_shared<DataTypeDateTime>(),
parseQuery(codec_parser, "(Delta(4), ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH),
parseQuery(codec_parser, "(Delta(4), ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS),
"Event time."
},
{
"metric",
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()),
parseQuery(codec_parser, "(ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH),
parseQuery(codec_parser, "(ZSTD(1))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS),
"Metric name."
},
{
"value",
std::make_shared<DataTypeFloat64>(),
parseQuery(codec_parser, "(ZSTD(3))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH),
parseQuery(codec_parser, "(ZSTD(3))", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS),
"Metric value."
}
};

5
src/Interpreters/DDLTask.cpp
View File

@ -154,7 +154,8 @@ void DDLLogEntry::parse(const String & data)
rb >> "settings: " >> settings_str >> "\n";
ParserSetQuery parser{true};
constexpr UInt64 max_depth = 16;
ASTPtr settings_ast = parseQuery(parser, settings_str, Context::getGlobalContextInstance()->getSettingsRef().max_query_size, max_depth);
constexpr UInt64 max_backtracks = DBMS_DEFAULT_MAX_PARSER_BACKTRACKS;
ASTPtr settings_ast = parseQuery(parser, settings_str, Context::getGlobalContextInstance()->getSettingsRef().max_query_size, max_depth, max_backtracks);
settings.emplace(std::move(settings_ast->as<ASTSetQuery>()->changes));
}
}
@ -197,7 +198,7 @@ void DDLTaskBase::parseQueryFromEntry(ContextPtr context)
ParserQuery parser_query(end, settings.allow_settings_after_format_in_insert);
String description;
query = parseQuery(parser_query, begin, end, description, 0, settings.max_parser_depth);
query = parseQuery(parser_query, begin, end, description, 0, settings.max_parser_depth, settings.max_parser_backtracks);
}
void DDLTaskBase::formatRewrittenQuery(ContextPtr context)

2
src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
View File

@ -96,7 +96,7 @@ static ASTPtr parseAdditionalPostFilter(const Context & context)
ParserExpression parser;
return parseQuery(
parser, filter.data(), filter.data() + filter.size(),
"additional filter", settings.max_query_size, settings.max_parser_depth);
"additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
static ActionsDAGPtr makeAdditionalPostFilter(ASTPtr & ast, ContextPtr context, const Block & header)

10
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -381,7 +381,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns)
String type_name = column.type->getName();
const char * pos = type_name.data();
const char * end = pos + type_name.size();
column_declaration->type = parseQuery(type_parser, pos, end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
column_declaration->type = parseQuery(type_parser, pos, end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
columns_list->children.emplace_back(column_declaration);
}
@ -401,7 +401,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns,
String type_name = alias_column.type->getName();
const char * type_pos = type_name.data();
const char * type_end = type_pos + type_name.size();
column_declaration->type = parseQuery(type_parser, type_pos, type_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
column_declaration->type = parseQuery(type_parser, type_pos, type_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
column_declaration->default_specifier = "ALIAS";
@ -409,7 +409,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns,
const char * alias_pos = alias.data();
const char * alias_end = alias_pos + alias.size();
ParserExpression expression_parser;
column_declaration->default_expression = parseQuery(expression_parser, alias_pos, alias_end, "expression", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
column_declaration->default_expression = parseQuery(expression_parser, alias_pos, alias_end, "expression", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
column_declaration->children.push_back(column_declaration->default_expression);
columns_list->children.emplace_back(column_declaration);
@ -433,7 +433,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
String type_name = column.type->getName();
const char * type_name_pos = type_name.data();
const char * type_name_end = type_name_pos + type_name.size();
column_declaration->type = parseQuery(type_parser, type_name_pos, type_name_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
column_declaration->type = parseQuery(type_parser, type_name_pos, type_name_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
if (column.default_desc.expression)
{
@ -1852,10 +1852,12 @@ void InterpreterCreateQuery::addColumnsDescriptionToCreateQueryIfNecessary(ASTCr
auto ast_storage = std::make_shared<ASTStorage>();
unsigned max_parser_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
unsigned max_parser_backtracks = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_backtracks);
auto query_from_storage = DB::getCreateQueryFromStorage(storage,
ast_storage,
false,
max_parser_depth,
max_parser_backtracks,
true);
auto & create_query_from_storage = query_from_storage->as<ASTCreateQuery &>();

3
src/Interpreters/InterpreterDeleteQuery.cpp
View File

@ -97,7 +97,8 @@ BlockIO InterpreterDeleteQuery::execute()
alter_query.data() + alter_query.size(),
"ALTER query",
0,
DBMS_DEFAULT_MAX_PARSER_DEPTH);
DBMS_DEFAULT_MAX_PARSER_DEPTH,
DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
auto context = Context::createCopy(getContext());
context->setSetting("mutations_sync", 2); /// Lightweight delete is always synchronous

2
src/Interpreters/InterpreterKillQueryQuery.cpp
View File

@ -281,7 +281,7 @@ BlockIO InterpreterKillQueryQuery::execute()
const auto with_round_bracket = alter_command.front() == '(';
ParserAlterCommand parser{with_round_bracket};
auto command_ast
= parseQuery(parser, alter_command, 0, getContext()->getSettingsRef().max_parser_depth);
= parseQuery(parser, alter_command, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks);
required_access_rights = InterpreterAlterQuery::getRequiredAccessForCommand(
command_ast->as<const ASTAlterCommand &>(), table_id.database_name, table_id.table_name);
if (!access->isGranted(required_access_rights))

4
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -160,7 +160,7 @@ FilterDAGInfoPtr generateFilterActions(
{
ParserExpression expr_parser;
/// We should add back quotes around column name as it can contain dots.
expr_list->children.push_back(parseQuery(expr_parser, backQuoteIfNeed(column_str), 0, context->getSettingsRef().max_parser_depth));
expr_list->children.push_back(parseQuery(expr_parser, backQuoteIfNeed(column_str), 0, context->getSettingsRef().max_parser_depth, context->getSettingsRef().max_parser_backtracks));
}
select_ast->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared<ASTTablesInSelectQuery>());
@ -331,7 +331,7 @@ ASTPtr parseAdditionalFilterConditionForTable(
const auto & settings = context.getSettingsRef();
return parseQuery(
parser, filter.data(), filter.data() + filter.size(),
"additional filter", settings.max_query_size, settings.max_parser_depth);
"additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
}

2
src/Interpreters/JoinToSubqueryTransformVisitor.cpp
View File

@ -43,7 +43,7 @@ ASTPtr makeSubqueryTemplate(const String & table_alias)
String query_template = "(select * from _t)";
if (!table_alias.empty())
query_template += " as " + table_alias;
ASTPtr subquery_template = parseQuery(parser, query_template, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
ASTPtr subquery_template = parseQuery(parser, query_template, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
if (!subquery_template)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot parse subquery template");
return subquery_template;

2
src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp
View File

@ -19,7 +19,7 @@ using namespace DB;
static inline ASTPtr tryRewrittenCreateQuery(const String & query, ContextPtr context)
{
ParserExternalDDLQuery external_ddl_parser;
ASTPtr ast = parseQuery(external_ddl_parser, "EXTERNAL DDL FROM MySQL(test_database, test_database) " + query, 0, 0);
ASTPtr ast = parseQuery(external_ddl_parser, "EXTERNAL DDL FROM MySQL(test_database, test_database) " + query, 0, 0, 0);
return MySQLInterpreter::InterpreterCreateImpl::getRewrittenQueries(
*ast->as<ASTExternalDDLQuery>()->external_ddl->as<MySQLParser::ASTCreateQuery>(),

4
src/Interpreters/SystemLog.cpp
View File

@ -216,7 +216,7 @@ std::shared_ptr<TSystemLog> createSystemLog(
/// Validate engine definition syntax to prevent some configuration errors.
ParserStorageWithComment storage_parser;
auto storage_ast = parseQuery(storage_parser, log_settings.engine.data(), log_settings.engine.data() + log_settings.engine.size(),
"Storage to create table for " + config_prefix, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
"Storage to create table for " + config_prefix, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
auto & storage_with_comment = storage_ast->as<StorageWithComment &>();
/// Add comment to AST. So it will be saved when the table will be renamed.
@ -647,7 +647,7 @@ ASTPtr SystemLog<LogElement>::getCreateTableQuery()
ASTPtr storage_with_comment_ast = parseQuery(
storage_parser, storage_def.data(), storage_def.data() + storage_def.size(),
"Storage to create table for " + LogElement::name(), 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
"Storage to create table for " + LogElement::name(), 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
StorageWithComment & storage_with_comment = storage_with_comment_ast->as<StorageWithComment &>();

10
src/Interpreters/executeQuery.cpp
View File

@ -746,18 +746,18 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
{
ParserKQLStatement parser(end, settings.allow_settings_after_format_in_insert);
/// TODO: parser should fail early when max_query_size limit is reached.
ast = parseKQLQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
ast = parseKQLQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
else if (settings.dialect == Dialect::prql && !internal)
{
ParserPRQLQuery parser(max_query_size, settings.max_parser_depth);
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
ParserPRQLQuery parser(max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
else
{
ParserQuery parser(end, settings.allow_settings_after_format_in_insert);
/// TODO: parser should fail early when max_query_size limit is reached.
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
#ifndef NDEBUG
/// Verify that AST formatting is consistent:
@ -774,7 +774,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
ast2 = parseQuery(parser,
formatted1.data(),
formatted1.data() + formatted1.size(),
"", new_max_query_size, settings.max_parser_depth);
"", new_max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
catch (const Exception & e)
{

2
src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp
View File

@ -122,7 +122,7 @@ ASTPtr parseCustomKeyForTable(const String & custom_key, const Context & context
const auto & settings = context.getSettingsRef();
return parseQuery(
parser, custom_key.data(), custom_key.data() + custom_key.size(),
"parallel replicas custom key", settings.max_query_size, settings.max_parser_depth);
"parallel replicas custom key", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
}
}

4
src/Interpreters/loadMetadata.cpp
View File

@ -55,9 +55,11 @@ static void executeCreateQuery(
bool create,
bool has_force_restore_data_flag)
{
const Settings & settings = context->getSettingsRef();
ParserCreateQuery parser;
ASTPtr ast = parseQuery(
parser, query.data(), query.data() + query.size(), "in file " + file_name, 0, context->getSettingsRef().max_parser_depth);
parser, query.data(), query.data() + query.size(), "in file " + file_name,
0, settings.max_parser_depth, settings.max_parser_backtracks);
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.setDatabase(database);

4
src/Interpreters/parseColumnsListForTableFunction.cpp
View File

@ -115,7 +115,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con
ParserColumnDeclarationList parser(true, true);
const Settings & settings = context->getSettingsRef();
ASTPtr columns_list_raw = parseQuery(parser, structure, "columns declaration list", settings.max_query_size, settings.max_parser_depth);
ASTPtr columns_list_raw = parseQuery(parser, structure, "columns declaration list", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
auto * columns_list = dynamic_cast<ASTExpressionList *>(columns_list_raw.get());
if (!columns_list)
@ -136,7 +136,7 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip
const char * start = structure.data();
const char * end = structure.data() + structure.size();
ASTPtr columns_list_raw = tryParseQuery(
parser, start, end, error, false, "columns declaration list", false, settings.max_query_size, settings.max_parser_depth);
parser, start, end, error, false, "columns declaration list", false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks, true);
if (!columns_list_raw)
return false;

2
src/Interpreters/tests/gtest_comparison_graph.cpp
View File

@ -12,7 +12,7 @@ using namespace DB;
static ComparisonGraph<ASTPtr> getGraph(const String & query)
{
ParserExpressionList parser(false);
ASTPtr ast = parseQuery(parser, query, 0, 0);
ASTPtr ast = parseQuery(parser, query, 0, 0, 0);
return ComparisonGraph<ASTPtr>(ast->children);
}

10
src/Interpreters/tests/gtest_cycle_aliases.cpp
View File

@ -14,10 +14,10 @@ TEST(QueryNormalizer, SimpleLoopAlias)
{
String query = "a as a";
ParserExpressionList parser(false);
ASTPtr ast = parseQuery(parser, query, 0, 0);
ASTPtr ast = parseQuery(parser, query, 0, 0, 0);
Aliases aliases;
aliases["a"] = parseQuery(parser, "a as a", 0, 0)->children[0];
aliases["a"] = parseQuery(parser, "a as a", 0, 0, 0)->children[0];
Settings settings;
QueryNormalizer::Data normalizer_data(aliases, {}, false, settings, false);
@ -28,11 +28,11 @@ TEST(QueryNormalizer, SimpleCycleAlias)
{
String query = "a as b, b as a";
ParserExpressionList parser(false);
ASTPtr ast = parseQuery(parser, query, 0, 0);
ASTPtr ast = parseQuery(parser, query, 0, 0, 0);
Aliases aliases;
aliases["a"] = parseQuery(parser, "b as a", 0, 0)->children[0];
aliases["b"] = parseQuery(parser, "a as b", 0, 0)->children[0];
aliases["a"] = parseQuery(parser, "b as a", 0, 0, 0)->children[0];
aliases["b"] = parseQuery(parser, "a as b", 0, 0, 0)->children[0];
Settings settings;
QueryNormalizer::Data normalizer_data(aliases, {}, false, settings, true);

4
src/Interpreters/tests/gtest_table_overrides.cpp
View File

@ -34,11 +34,11 @@ TEST_P(TableOverrideTest, applyOverrides)
const auto & [database_query, table_query, expected_query] = GetParam();
ParserCreateQuery parser;
ASTPtr database_ast;
ASSERT_NO_THROW(database_ast = parseQuery(parser, database_query, 0, 0));
ASSERT_NO_THROW(database_ast = parseQuery(parser, database_query, 0, 0, 0));
auto * database = database_ast->as<ASTCreateQuery>();
ASSERT_NE(nullptr, database);
ASTPtr table_ast;
ASSERT_NO_THROW(table_ast = parseQuery(parser, table_query, 0, 0));
ASSERT_NO_THROW(table_ast = parseQuery(parser, table_query, 0, 0, 0));
auto * table = table_ast->as<ASTCreateQuery>();
ASSERT_NE(nullptr, table);
auto table_name = table->table->as<ASTIdentifier>()->name();

2
src/Parsers/ExpressionListParsers.cpp
View File

@ -1918,7 +1918,7 @@ public:
&& string_literal->as<ASTLiteral &>().value.tryGet(literal))
{
Tokens tokens(literal.data(), literal.data() + literal.size());
IParser::Pos token_pos(tokens, 0);
IParser::Pos token_pos(tokens, pos.max_depth, pos.max_backtracks);
Expected token_expected;
ASTPtr expr;

33
src/Parsers/IParser.cpp Normal file
View File

@ -0,0 +1,33 @@
#include <Parsers/IParser.h>
#include <iostream>
namespace DB
{
IParser::Pos & IParser::Pos::operator=(const IParser::Pos & rhs)
{
depth = rhs.depth;
max_depth = rhs.max_depth;
if (rhs.backtracks > backtracks)
backtracks = rhs.backtracks;
max_backtracks = rhs.max_backtracks;
if (rhs < *this)
{
++backtracks;
if (max_backtracks && backtracks > max_backtracks)
throw Exception(ErrorCodes::TOO_DEEP_RECURSION, "Maximum amount of backtracking ({}) exceeded in the parser. "
"Consider rising max_parser_backtracks parameter.", max_backtracks);
}
TokenIterator::operator=(rhs);
if (backtracks % 1000 == 0)
std::cerr << backtracks << "\n";
return *this;
}
}

15
src/Parsers/IParser.h
View File

@ -62,11 +62,18 @@ public:
uint32_t depth = 0;
uint32_t max_depth = 0;
Pos(Tokens & tokens_, uint32_t max_depth_) : TokenIterator(tokens_), max_depth(max_depth_)
uint32_t backtracks = 0;
uint32_t max_backtracks = 0;
Pos(Tokens & tokens_, uint32_t max_depth_, uint32_t max_backtracks_)
: TokenIterator(tokens_), max_depth(max_depth_), max_backtracks(max_backtracks_)
{
}
Pos(TokenIterator token_iterator_, uint32_t max_depth_) : TokenIterator(token_iterator_), max_depth(max_depth_) { }
Pos(TokenIterator token_iterator_, uint32_t max_depth_, uint32_t max_backtracks_)
: TokenIterator(token_iterator_), max_depth(max_depth_), max_backtracks(max_backtracks_)
{
}
ALWAYS_INLINE void increaseDepth()
{
@ -97,6 +104,10 @@ public:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error in parser: incorrect calculation of parse depth");
--depth;
}
Pos(const Pos & rhs) = default;
Pos & operator=(const Pos & rhs);
};
/** Get the text of this parser parses. */

8
src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp
View File

@ -279,13 +279,13 @@ String IParserKQLFunction::getKQLFunctionName(IParser::Pos & pos)
}
String IParserKQLFunction::kqlCallToExpression(
const std::string_view function_name, const std::initializer_list<const std::string_view> params, const uint32_t max_depth)
const std::string_view function_name, const std::initializer_list<const std::string_view> params, uint32_t max_depth, uint32_t max_backtracks)
{
return kqlCallToExpression(function_name, std::span(params), max_depth);
return kqlCallToExpression(function_name, std::span(params), max_depth, max_backtracks);
}
String IParserKQLFunction::kqlCallToExpression(
const std::string_view function_name, const std::span<const std::string_view> params, const uint32_t max_depth)
const std::string_view function_name, const std::span<const std::string_view> params, uint32_t max_depth, uint32_t max_backtracks)
{
const auto params_str = std::accumulate(
std::cbegin(params),
@ -302,7 +302,7 @@ String IParserKQLFunction::kqlCallToExpression(
const auto kql_call = std::format("{}({})", function_name, params_str);
DB::Tokens call_tokens(kql_call.c_str(), kql_call.c_str() + kql_call.length());
DB::IParser::Pos tokens_pos(call_tokens, max_depth);
DB::IParser::Pos tokens_pos(call_tokens, max_depth, max_backtracks);
return DB::IParserKQLFunction::getExpression(tokens_pos);
}

4
src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h
View File

@ -77,8 +77,8 @@ public:
static std::optional<String>
getOptionalArgument(const String & function_name, DB::IParser::Pos & pos, ArgumentState argument_state = ArgumentState::Parsed);
static String
kqlCallToExpression(std::string_view function_name, std::initializer_list<const std::string_view> params, uint32_t max_depth);
static String kqlCallToExpression(std::string_view function_name, std::span<const std::string_view> params, uint32_t max_depth);
kqlCallToExpression(std::string_view function_name, std::initializer_list<const std::string_view> params, uint32_t max_depth, uint32_t max_backtracks);
static String kqlCallToExpression(std::string_view function_name, std::span<const std::string_view> params, uint32_t max_depth, uint32_t max_backtracks);
static String escapeSingleQuotes(const String & input);
protected:

2
src/Parsers/Kusto/KustoFunctions/KQLCastingFunctions.cpp
View File

@ -99,7 +99,7 @@ bool ToTimeSpan::convertImpl(String & out, IParser::Pos & pos)
++pos;
try
{
auto result = kqlCallToExpression("time", {arg}, pos.max_depth);
auto result = kqlCallToExpression("time", {arg}, pos.max_depth, pos.max_backtracks);
out = std::format("{}", result);
}
catch (...)

10
src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp
View File

@ -99,7 +99,7 @@ bool ArrayRotateRight::convertImpl(String & out, IParser::Pos & pos)
const auto array = getArgument(function_name, pos, ArgumentState::Raw);
const auto count = getArgument(function_name, pos, ArgumentState::Raw);
out = kqlCallToExpression("array_rotate_left", {array, "-1 * " + count}, pos.max_depth);
out = kqlCallToExpression("array_rotate_left", {array, "-1 * " + count}, pos.max_depth, pos.max_backtracks);
return true;
}
@ -140,7 +140,7 @@ bool ArrayShiftRight::convertImpl(String & out, IParser::Pos & pos)
"array_shift_left",
fill ? std::initializer_list<std::string_view>{array, negated_count, *fill}
: std::initializer_list<std::string_view>{array, negated_count},
pos.max_depth);
pos.max_depth, pos.max_backtracks);
return true;
}
@ -233,8 +233,8 @@ bool JaccardIndex::convertImpl(String & out, IParser::Pos & pos)
const auto rhs = getArgument(function_name, pos, ArgumentState::Raw);
out = std::format(
"divide(length({0}), length({1}))",
kqlCallToExpression("set_intersect", {lhs, rhs}, pos.max_depth),
kqlCallToExpression("set_union", {lhs, rhs}, pos.max_depth));
kqlCallToExpression("set_intersect", {lhs, rhs}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("set_union", {lhs, rhs}, pos.max_depth, pos.max_backtracks));
return true;
}
@ -292,7 +292,7 @@ bool SetDifference::convertImpl(String & out, IParser::Pos & pos)
while (auto next_array = getOptionalArgument(function_name, pos, ArgumentState::Raw))
arrays.push_back(*next_array);
return kqlCallToExpression("set_union", std::vector<std::string_view>(arrays.cbegin(), arrays.cend()), pos.max_depth);
return kqlCallToExpression("set_union", std::vector<std::string_view>(arrays.cbegin(), arrays.cend()), pos.max_depth, pos.max_backtracks);
});
out = std::format("arrayFilter(x -> not has({1}, x), arrayDistinct({0}))", lhs, rhs);

32
src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp
View File

@ -34,10 +34,10 @@ bool Ipv4Compare::convertImpl(String & out, IParser::Pos & pos)
"sign(IPv4StringToNumOrNull(toString((tupleElement(IPv4CIDRToRange(assumeNotNull(lhs_ip_{5}), "
"toUInt8(min2({4}, min2(assumeNotNull(lhs_mask_{5}), assumeNotNull(rhs_mask_{5})))) as mask_{5}), 1))))"
" - IPv4StringToNumOrNull(toString((tupleElement(IPv4CIDRToRange(assumeNotNull(rhs_ip_{5}), mask_{5}), 1))))))",
kqlCallToExpression("parse_ipv4", {lhs}, pos.max_depth),
kqlCallToExpression("ipv4_netmask_suffix", {lhs}, pos.max_depth),
kqlCallToExpression("parse_ipv4", {rhs}, pos.max_depth),
kqlCallToExpression("ipv4_netmask_suffix", {rhs}, pos.max_depth),
kqlCallToExpression("parse_ipv4", {lhs}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("ipv4_netmask_suffix", {lhs}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("parse_ipv4", {rhs}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("ipv4_netmask_suffix", {rhs}, pos.max_depth, pos.max_backtracks),
mask ? *mask : "32",
generateUniqueIdentifier());
return true;
@ -56,8 +56,8 @@ bool Ipv4IsInRange::convertImpl(String & out, IParser::Pos & pos)
"or isNull({1} as range_start_ip_{3}) or isNull({2} as range_mask_{3}), null, "
"bitXor(range_start_ip_{3}, bitAnd(ip_{3}, bitNot(toUInt32(intExp2(toInt32(32 - range_mask_{3})) - 1)))) = 0) ",
ip_address,
kqlCallToExpression("parse_ipv4", {ip_range}, pos.max_depth),
kqlCallToExpression("ipv4_netmask_suffix", {ip_range}, pos.max_depth),
kqlCallToExpression("parse_ipv4", {ip_range}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("ipv4_netmask_suffix", {ip_range}, pos.max_depth, pos.max_backtracks),
generateUniqueIdentifier());
return true;
}
@ -71,7 +71,7 @@ bool Ipv4IsMatch::convertImpl(String & out, IParser::Pos & pos)
const auto lhs = getArgument(function_name, pos, ArgumentState::Raw);
const auto rhs = getArgument(function_name, pos, ArgumentState::Raw);
const auto mask = getOptionalArgument(function_name, pos, ArgumentState::Raw);
out = std::format("equals({}, 0)", kqlCallToExpression("ipv4_compare", {lhs, rhs, mask ? *mask : "32"}, pos.max_depth));
out = std::format("equals({}, 0)", kqlCallToExpression("ipv4_compare", {lhs, rhs, mask ? *mask : "32"}, pos.max_depth, pos.max_backtracks));
return true;
}
@ -196,7 +196,7 @@ bool Ipv6IsMatch::convertImpl(String & out, IParser::Pos & pos)
const auto lhs = getArgument(function_name, pos, ArgumentState::Raw);
const auto rhs = getArgument(function_name, pos, ArgumentState::Raw);
const auto mask = getOptionalArgument(function_name, pos, ArgumentState::Raw);
out = std::format("equals({}, 0)", kqlCallToExpression("ipv6_compare", {lhs, rhs, mask ? *mask : "128"}, pos.max_depth));
out = std::format("equals({}, 0)", kqlCallToExpression("ipv6_compare", {lhs, rhs, mask ? *mask : "128"}, pos.max_depth, pos.max_backtracks));
return true;
}
@ -228,9 +228,9 @@ bool ParseIpv6Mask::convertImpl(String & out, IParser::Pos & pos)
const auto unique_identifier = generateUniqueIdentifier();
out = std::format(
"if(empty({0} as ipv4_{3}), {1}, {2})",
kqlCallToExpression("format_ipv4", {"trim_start('::', " + ip_address + ")", mask + " - 96"}, pos.max_depth),
kqlCallToExpression("parse_ipv6", {"strcat(tostring(parse_ipv6(" + ip_address + ")), '/', tostring(" + mask + "))"}, pos.max_depth),
kqlCallToExpression("parse_ipv6", {"ipv4_" + unique_identifier}, pos.max_depth),
kqlCallToExpression("format_ipv4", {"trim_start('::', " + ip_address + ")", mask + " - 96"}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("parse_ipv6", {"strcat(tostring(parse_ipv6(" + ip_address + ")), '/', tostring(" + mask + "))"}, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("parse_ipv6", {"ipv4_" + unique_identifier}, pos.max_depth, pos.max_backtracks),
unique_identifier);
return true;
}
@ -247,9 +247,9 @@ bool FormatIpv4::convertImpl(String & out, IParser::Pos & pos)
"ifNull(if(isNotNull(toUInt32OrNull(toString({0})) as param_as_uint32_{3}) and toTypeName({0}) = 'String' or ({1}) < 0 "
"or isNull(ifNull(param_as_uint32_{3}, {2}) as ip_as_number_{3}), null, "
"IPv4NumToString(bitAnd(ip_as_number_{3}, bitNot(toUInt32(intExp2(toInt32(32 - ({1}))) - 1))))), '')",
ParserKQLBase::getExprFromToken(ip_address, pos.max_depth),
ParserKQLBase::getExprFromToken(ip_address, pos.max_depth, pos.max_backtracks),
mask ? *mask : "32",
kqlCallToExpression("parse_ipv4", {"tostring(" + ip_address + ")"}, pos.max_depth),
kqlCallToExpression("parse_ipv4", {"tostring(" + ip_address + ")"}, pos.max_depth, pos.max_backtracks),
generateUniqueIdentifier());
return true;
}
@ -266,10 +266,10 @@ bool FormatIpv4Mask::convertImpl(String & out, IParser::Pos & pos)
out = std::format(
"if(empty({1} as formatted_ip_{2}) or position(toTypeName({0}), 'Int') = 0 or not {0} between 0 and 32, '', "
"concat(formatted_ip_{2}, '/', toString(toInt64(min2({0}, ifNull({3} as suffix_{2}, 32))))))",
ParserKQLBase::getExprFromToken(calculated_mask, pos.max_depth),
kqlCallToExpression("format_ipv4", {ip_address, calculated_mask}, pos.max_depth),
ParserKQLBase::getExprFromToken(calculated_mask, pos.max_depth, pos.max_backtracks),
kqlCallToExpression("format_ipv4", {ip_address, calculated_mask}, pos.max_depth, pos.max_backtracks),
generateUniqueIdentifier(),
kqlCallToExpression("ipv4_netmask_suffix", {"tostring(" + ip_address + ")"}, pos.max_depth));
kqlCallToExpression("ipv4_netmask_suffix", {"tostring(" + ip_address + ")"}, pos.max_depth, pos.max_backtracks));
return true;
}
}

4
src/Parsers/Kusto/KustoFunctions/KQLStringFunctions.cpp
View File

@ -442,7 +442,7 @@ bool ParseJSON::convertImpl(String & out, IParser::Pos & pos)
{
--pos;
auto arg = getArgument(fn_name, pos);
auto result = kqlCallToExpression("dynamic", {arg}, pos.max_depth);
auto result = kqlCallToExpression("dynamic", {arg}, pos.max_depth, pos.max_backtracks);
out = std::format("{}", result);
}
else
@ -729,7 +729,7 @@ bool Trim::convertImpl(String & out, IParser::Pos & pos)
const auto regex = getArgument(fn_name, pos, ArgumentState::Raw);
const auto source = getArgument(fn_name, pos, ArgumentState::Raw);
out = kqlCallToExpression("trim_start", {regex, std::format("trim_end({0}, {1})", regex, source)}, pos.max_depth);
out = kqlCallToExpression("trim_start", {regex, std::format("trim_end({0}, {1})", regex, source)}, pos.max_depth, pos.max_backtracks);
return true;
}

2
src/Parsers/Kusto/ParserKQLDistinct.cpp
View File

@ -12,7 +12,7 @@ bool ParserKQLDistinct::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
expr = getExprFromToken(pos);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(false).parse(new_pos, select_expression_list, expected))
return false;

4
src/Parsers/Kusto/ParserKQLExtend.cpp
View File

@ -23,7 +23,7 @@ bool ParserKQLExtend ::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
String except_str;
String new_extend_str;
Tokens ntokens(extend_expr.c_str(), extend_expr.c_str() + extend_expr.size());
IParser::Pos npos(ntokens, pos.max_depth);
IParser::Pos npos(ntokens, pos.max_depth, pos.max_backtracks);
String alias;
@ -77,7 +77,7 @@ bool ParserKQLExtend ::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
String expr = std::format("SELECT * {}, {} from prev", except_str, new_extend_str);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!ParserSelectQuery().parse(new_pos, select_query, expected))
return false;

2
src/Parsers/Kusto/ParserKQLFilter.cpp
View File

@ -14,7 +14,7 @@ bool ParserKQLFilter::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
ASTPtr where_expression;
Tokens token_filter(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos pos_filter(token_filter, pos.max_depth);
IParser::Pos pos_filter(token_filter, pos.max_depth, pos.max_backtracks);
if (!ParserExpressionWithOptionalAlias(false).parse(pos_filter, where_expression, expected))
return false;

2
src/Parsers/Kusto/ParserKQLLimit.cpp
View File

@ -14,7 +14,7 @@ bool ParserKQLLimit::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
auto expr = getExprFromToken(pos);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!ParserExpressionWithOptionalAlias(false).parse(new_pos, limit_length, expected))
return false;

16
src/Parsers/Kusto/ParserKQLMVExpand.cpp
View File

@ -69,7 +69,7 @@ bool ParserKQLMVExpand::parseColumnArrayExprs(ColumnArrayExprs & column_array_ex
auto add_columns = [&]
{
column_array_expr = getExprFromToken(String(expr_begin_pos->begin, expr_end_pos->end), pos.max_depth);
column_array_expr = getExprFromToken(String(expr_begin_pos->begin, expr_end_pos->end), pos.max_depth, pos.max_backtracks);
if (alias.empty())
{
@ -189,7 +189,7 @@ bool ParserKQLMVExpand::parserMVExpand(KQLMVExpand & kql_mv_expand, Pos & pos, E
return true;
}
bool ParserKQLMVExpand::genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_node, int32_t max_depth)
bool ParserKQLMVExpand::genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks)
{
String expand_str;
String cast_type_column_remove, cast_type_column_rename;
@ -253,7 +253,7 @@ bool ParserKQLMVExpand::genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_no
if (cast_type_column_remove.empty())
{
query = std::format("Select {} {} From {} {}", columns, extra_columns, input, expand_str);
if (!parseSQLQueryByString(std::make_unique<ParserSelectQuery>(), query, sub_query_node, max_depth))
if (!parseSQLQueryByString(std::make_unique<ParserSelectQuery>(), query, sub_query_node, max_depth, max_backtracks))
return false;
if (!setSubQuerySource(sub_query_node, select_node, false, false))
return false;
@ -262,14 +262,14 @@ bool ParserKQLMVExpand::genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_no
else
{
query = std::format("(Select {} {} From {} {})", columns, extra_columns, input, expand_str);
if (!parseSQLQueryByString(std::make_unique<ParserTablesInSelectQuery>(), query, sub_query_node, max_depth))
if (!parseSQLQueryByString(std::make_unique<ParserTablesInSelectQuery>(), query, sub_query_node, max_depth, max_backtracks))
return false;
if (!setSubQuerySource(sub_query_node, select_node, true, false))
return false;
select_node = std::move(sub_query_node);
auto rename_query = std::format("(Select * {}, {} From {})", cast_type_column_remove, cast_type_column_rename, "query");
if (!parseSQLQueryByString(std::make_unique<ParserTablesInSelectQuery>(), rename_query, sub_query_node, max_depth))
if (!parseSQLQueryByString(std::make_unique<ParserTablesInSelectQuery>(), rename_query, sub_query_node, max_depth, max_backtracks))
return false;
if (!setSubQuerySource(sub_query_node, select_node, true, true))
return false;
@ -277,7 +277,7 @@ bool ParserKQLMVExpand::genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_no
select_node = std::move(sub_query_node);
query = std::format("Select * {}, {} from {}", cast_type_column_restore, cast_type_column_restore_name, "rename_query");
if (!parseSQLQueryByString(std::make_unique<ParserSelectQuery>(), query, sub_query_node, max_depth))
if (!parseSQLQueryByString(std::make_unique<ParserSelectQuery>(), query, sub_query_node, max_depth, max_backtracks))
return false;
sub_query_node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::TABLES, std::move(select_node));
select_node = std::move(sub_query_node);
@ -294,12 +294,12 @@ bool ParserKQLMVExpand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
KQLMVExpand kql_mv_expand;
if (!parserMVExpand(kql_mv_expand, pos, expected))
return false;
if (!genQuery(kql_mv_expand, node, pos.max_depth))
if (!genQuery(kql_mv_expand, node, pos.max_depth, pos.max_backtracks))
return false;
const String setting_str = "enable_unaligned_array_join = 1";
Tokens token_settings(setting_str.c_str(), setting_str.c_str() + setting_str.size());
IParser::Pos pos_settings(token_settings, pos.max_depth);
IParser::Pos pos_settings(token_settings, pos.max_depth, pos.max_backtracks);
if (!ParserSetQuery(true).parse(pos_settings, setting, expected))
return false;

2
src/Parsers/Kusto/ParserKQLMVExpand.h
View File

@ -33,7 +33,7 @@ protected:
static bool parseColumnArrayExprs(ColumnArrayExprs & column_array_exprs, Pos & pos, Expected & expected);
static bool parserMVExpand(KQLMVExpand & kql_mv_expand, Pos & pos, Expected & expected);
static bool genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_node, int32_t max_depth);
static bool genQuery(KQLMVExpand & kql_mv_expand, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks);
const char * getName() const override { return "KQL mv-expand"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;

20
src/Parsers/Kusto/ParserKQLMakeSeries.cpp
View File

@ -142,7 +142,7 @@ bool ParserKQLMakeSeries ::parseFromToStepClause(FromToStepClause & from_to_step
|| ParserKQLDateTypeTimespan().parseConstKQLTimespan(from_to_step.step_str))
{
from_to_step.is_timespan = true;
from_to_step.step = std::stod(getExprFromToken(from_to_step.step_str, pos.max_depth));
from_to_step.step = std::stod(getExprFromToken(from_to_step.step_str, pos.max_depth, pos.max_backtracks));
}
else
from_to_step.step = std::stod(from_to_step.step_str);
@ -150,7 +150,7 @@ bool ParserKQLMakeSeries ::parseFromToStepClause(FromToStepClause & from_to_step
return true;
}
bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr & select_node, const uint32_t & max_depth)
bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks)
{
const uint64_t era_diff
= 62135596800; // this magic number is the differicen is second form 0001-01-01 (Azure start time ) and 1970-01-01 (CH start time)
@ -166,15 +166,15 @@ bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr &
auto step = from_to_step.step;
if (!kql_make_series.from_to_step.from_str.empty())
start_str = getExprFromToken(kql_make_series.from_to_step.from_str, max_depth);
start_str = getExprFromToken(kql_make_series.from_to_step.from_str, max_depth, max_backtracks);
if (!kql_make_series.from_to_step.to_str.empty())
end_str = getExprFromToken(from_to_step.to_str, max_depth);
end_str = getExprFromToken(from_to_step.to_str, max_depth, max_backtracks);
auto date_type_cast = [&](String & src)
{
Tokens tokens(src.c_str(), src.c_str() + src.size());
IParser::Pos pos(tokens, max_depth);
IParser::Pos pos(tokens, max_depth, max_backtracks);
String res;
while (isValidKQLPos(pos))
{
@ -201,7 +201,7 @@ bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr &
{
std::vector<String> group_expression_tokens;
Tokens tokens(group_expression.c_str(), group_expression.c_str() + group_expression.size());
IParser::Pos pos(tokens, max_depth);
IParser::Pos pos(tokens, max_depth, max_backtracks);
while (isValidKQLPos(pos))
{
if (String(pos->begin, pos->end) == "AS")
@ -296,7 +296,7 @@ bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr &
ASTPtr sub_query_node;
if (!ParserSimpleCHSubquery(select_node).parseByString(sub_sub_query, sub_query_node, max_depth))
if (!ParserSimpleCHSubquery(select_node).parseByString(sub_sub_query, sub_query_node, max_depth, max_backtracks))
return false;
select_node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::TABLES, std::move(sub_query_node));
@ -351,7 +351,7 @@ bool ParserKQLMakeSeries ::makeSeries(KQLMakeSeries & kql_make_series, ASTPtr &
else
main_query = std::format("{},{}", group_expression_alias, final_axis_agg_alias_list);
if (!ParserSimpleCHSubquery(select_node).parseByString(sub_query, sub_query_node, max_depth))
if (!ParserSimpleCHSubquery(select_node).parseByString(sub_query, sub_query_node, max_depth, max_backtracks))
return false;
select_node->as<ASTSelectQuery>()->setExpression(ASTSelectQuery::Expression::TABLES, std::move(sub_query_node));
@ -411,10 +411,10 @@ bool ParserKQLMakeSeries ::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
subquery_columns += ", " + column_str;
}
makeSeries(kql_make_series, node, pos.max_depth);
makeSeries(kql_make_series, node, pos.max_depth, pos.max_backtracks);
Tokens token_main_query(kql_make_series.main_query.c_str(), kql_make_series.main_query.c_str() + kql_make_series.main_query.size());
IParser::Pos pos_main_query(token_main_query, pos.max_depth);
IParser::Pos pos_main_query(token_main_query, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(true).parse(pos_main_query, select_expression_list, expected))
return false;

2
src/Parsers/Kusto/ParserKQLMakeSeries.h
View File

@ -42,7 +42,7 @@ protected:
String main_query;
};
static bool makeSeries(KQLMakeSeries & kql_make_series, ASTPtr & select_node, const uint32_t & max_depth);
static bool makeSeries(KQLMakeSeries & kql_make_series, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks);
static bool parseAggregationColumns(AggregationColumns & aggregation_columns, Pos & pos);
static bool parseFromToStepClause(FromToStepClause & from_to_step, Pos & pos);

2
src/Parsers/Kusto/ParserKQLPrint.cpp
View File

@ -10,7 +10,7 @@ bool ParserKQLPrint::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
const String expr = getExprFromToken(pos);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(true).parse(new_pos, select_expression_list, expected))
return false;

2
src/Parsers/Kusto/ParserKQLProject.cpp
View File

@ -12,7 +12,7 @@ bool ParserKQLProject ::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
expr = getExprFromToken(pos);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(false).parse(new_pos, select_expression_list, expected))
return false;

18
src/Parsers/Kusto/ParserKQLQuery.cpp
View File

@ -33,20 +33,20 @@ namespace ErrorCodes
extern const int SYNTAX_ERROR;
}
bool ParserKQLBase::parseByString(const String expr, ASTPtr & node, const uint32_t max_depth)
bool ParserKQLBase::parseByString(String expr, ASTPtr & node, uint32_t max_depth, uint32_t max_backtracks)
{
Expected expected;
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos pos(tokens, max_depth);
IParser::Pos pos(tokens, max_depth, max_backtracks);
return parse(pos, node, expected);
}
bool ParserKQLBase::parseSQLQueryByString(ParserPtr && parser, String & query, ASTPtr & select_node, int32_t max_depth)
bool ParserKQLBase::parseSQLQueryByString(ParserPtr && parser, String & query, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks)
{
Expected expected;
Tokens token_subquery(query.c_str(), query.c_str() + query.size());
IParser::Pos pos_subquery(token_subquery, max_depth);
IParser::Pos pos_subquery(token_subquery, max_depth, max_backtracks);
if (!parser->parse(pos_subquery, select_node, expected))
return false;
return true;
@ -121,10 +121,10 @@ bool ParserKQLBase::setSubQuerySource(ASTPtr & select_query, ASTPtr & source, bo
return true;
}
String ParserKQLBase::getExprFromToken(const String & text, const uint32_t max_depth)
String ParserKQLBase::getExprFromToken(const String & text, uint32_t max_depth, uint32_t max_backtracks)
{
Tokens tokens(text.c_str(), text.c_str() + text.size());
IParser::Pos pos(tokens, max_depth);
IParser::Pos pos(tokens, max_depth, max_backtracks);
return getExprFromToken(pos);
}
@ -523,7 +523,7 @@ bool ParserKQLQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
String sub_query = std::format("({})", String(operation_pos.front().second->begin, last_pos->end));
Tokens token_subquery(sub_query.c_str(), sub_query.c_str() + sub_query.size());
IParser::Pos pos_subquery(token_subquery, pos.max_depth);
IParser::Pos pos_subquery(token_subquery, pos.max_depth, pos.max_backtracks);
if (!ParserKQLSubquery().parse(pos_subquery, tables, expected))
return false;
@ -544,7 +544,7 @@ bool ParserKQLQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
if (oprator)
{
Tokens token_clause(op_calsue.c_str(), op_calsue.c_str() + op_calsue.size());
IParser::Pos pos_clause(token_clause, pos.max_depth);
IParser::Pos pos_clause(token_clause, pos.max_depth, pos.max_backtracks);
if (!oprator->parse(pos_clause, node, expected))
return false;
}
@ -577,7 +577,7 @@ bool ParserKQLQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
auto expr = String("*");
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
if (!std::make_unique<ParserKQLProject>()->parse(new_pos, node, expected))
return false;
}

6
src/Parsers/Kusto/ParserKQLQuery.h
View File

@ -9,11 +9,11 @@ class ParserKQLBase : public IParserBase
{
public:
static String getExprFromToken(Pos & pos);
static String getExprFromToken(const String & text, uint32_t max_depth);
static String getExprFromToken(const String & text, uint32_t max_depth, uint32_t max_backtracks);
static String getExprFromPipe(Pos & pos);
static bool setSubQuerySource(ASTPtr & select_query, ASTPtr & source, bool dest_is_subquery, bool src_is_subquery);
static bool parseSQLQueryByString(ParserPtr && parser, String & query, ASTPtr & select_node, int32_t max_depth);
bool parseByString(String expr, ASTPtr & node, uint32_t max_depth);
static bool parseSQLQueryByString(ParserPtr && parser, String & query, ASTPtr & select_node, uint32_t max_depth, uint32_t max_backtracks);
bool parseByString(String expr, ASTPtr & node, uint32_t max_depth, uint32_t max_backtracks);
};
class ParserKQLQuery : public IParserBase

2
src/Parsers/Kusto/ParserKQLSort.cpp
View File

@ -19,7 +19,7 @@ bool ParserKQLSort::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
auto expr = getExprFromToken(pos);
Tokens tokens(expr.c_str(), expr.c_str() + expr.size());
IParser::Pos new_pos(tokens, pos.max_depth);
IParser::Pos new_pos(tokens, pos.max_depth, pos.max_backtracks);
auto pos_backup = new_pos;
if (!order_list.parse(pos_backup, order_expression_list, expected))

2
src/Parsers/Kusto/ParserKQLStatement.cpp
View File

@ -95,7 +95,7 @@ bool ParserKQLTableFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
}
++pos;
Tokens token_kql(kql_statement.c_str(), kql_statement.c_str() + kql_statement.size());
IParser::Pos pos_kql(token_kql, pos.max_depth);
IParser::Pos pos_kql(token_kql, pos.max_depth, pos.max_backtracks);
if (kql_p.parse(pos_kql, select, expected))
{

8
src/Parsers/Kusto/ParserKQLSummarize.cpp
View File

@ -192,10 +192,10 @@ bool ParserKQLSummarize::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
expr_columns = expr_columns + "," + expr_aggregation;
}
String converted_columns = getExprFromToken(expr_columns, pos.max_depth);
String converted_columns = getExprFromToken(expr_columns, pos.max_depth, pos.max_backtracks);
Tokens token_converted_columns(converted_columns.c_str(), converted_columns.c_str() + converted_columns.size());
IParser::Pos pos_converted_columns(token_converted_columns, pos.max_depth);
IParser::Pos pos_converted_columns(token_converted_columns, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(true).parse(pos_converted_columns, select_expression_list, expected))
return false;
@ -204,10 +204,10 @@ bool ParserKQLSummarize::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
if (groupby)
{
String converted_groupby = getExprFromToken(expr_groupby, pos.max_depth);
String converted_groupby = getExprFromToken(expr_groupby, pos.max_depth, pos.max_backtracks);
Tokens token_converted_groupby(converted_groupby.c_str(), converted_groupby.c_str() + converted_groupby.size());
IParser::Pos postoken_converted_groupby(token_converted_groupby, pos.max_depth);
IParser::Pos postoken_converted_groupby(token_converted_groupby, pos.max_depth, pos.max_backtracks);
if (!ParserNotEmptyExpressionList(false).parse(postoken_converted_groupby, group_expression_list, expected))
return false;

23
src/Parsers/Kusto/parseKQLQuery.cpp
View File

@ -322,12 +322,13 @@ ASTPtr tryParseKQLQuery(
bool allow_multi_statements,
size_t max_query_size,
size_t max_parser_depth,
size_t max_parser_backtracks,
bool skip_insignificant)
{
const char * query_begin = _out_query_end;
Tokens tokens(query_begin, all_queries_end, max_query_size, skip_insignificant);
/// NOTE: consider use UInt32 for max_parser_depth setting.
IParser::Pos token_iterator(tokens, static_cast<uint32_t>(max_parser_depth));
IParser::Pos token_iterator(tokens, static_cast<uint32_t>(max_parser_depth), static_cast<uint32_t>(max_parser_backtracks));
if (token_iterator->isEnd()
|| token_iterator->type == TokenType::Semicolon)
@ -441,10 +442,11 @@ ASTPtr parseKQLQueryAndMovePosition(
const std::string & query_description,
bool allow_multi_statements,
size_t max_query_size,
size_t max_parser_depth)
size_t max_parser_depth,
size_t max_parser_backtracks)
{
std::string error_message;
ASTPtr res = tryParseKQLQuery(parser, pos, end, error_message, false, query_description, allow_multi_statements, max_query_size, max_parser_depth);
ASTPtr res = tryParseKQLQuery(parser, pos, end, error_message, false, query_description, allow_multi_statements, max_query_size, max_parser_depth, max_parser_backtracks);
if (res)
return res;
@ -458,9 +460,10 @@ ASTPtr parseKQLQuery(
const char * end,
const std::string & query_description,
size_t max_query_size,
size_t max_parser_depth)
size_t max_parser_depth,
size_t max_parser_backtracks)
{
return parseKQLQueryAndMovePosition(parser, begin, end, query_description, false, max_query_size, max_parser_depth);
return parseKQLQueryAndMovePosition(parser, begin, end, query_description, false, max_query_size, max_parser_depth, max_parser_backtracks);
}
ASTPtr parseKQLQuery(
@ -468,18 +471,20 @@ ASTPtr parseKQLQuery(
const std::string & query,
const std::string & query_description,
size_t max_query_size,
size_t max_parser_depth)
size_t max_parser_depth,
size_t max_parser_backtracks)
{
return parseKQLQuery(parser, query.data(), query.data() + query.size(), query_description, max_query_size, max_parser_depth);
return parseKQLQuery(parser, query.data(), query.data() + query.size(), query_description, max_query_size, max_parser_depth, max_parser_backtracks);
}
ASTPtr parseKQLQuery(
IParser & parser,
const std::string & query,
size_t max_query_size,
size_t max_parser_depth)
size_t max_parser_depth,
size_t max_parser_backtracks)
{
return parseKQLQuery(parser, query.data(), query.data() + query.size(), parser.getName(), max_query_size, max_parser_depth);
return parseKQLQuery(parser, query.data(), query.data() + query.size(), parser.getName(), max_query_size, max_parser_depth, max_parser_backtracks);
}
}

26
src/Parsers/Kusto/parseKQLQuery.h
View File

@ -3,6 +3,7 @@
#include <Parsers/IAST_fwd.h>
#include <Parsers/parseQuery.h>
#include <IO/WriteBufferFromString.h>
namespace DB
{
@ -10,10 +11,6 @@ namespace DB
* Used in syntax error message.
*/
}
namespace DB
{
class IParser;
/// Parse query or set 'out_error_message'.
@ -24,11 +21,11 @@ ASTPtr tryParseKQLQuery(
std::string & out_error_message,
bool hilite,
const std::string & description,
bool allow_multi_statements, /// If false, check for non-space characters after semicolon and set error message if any.
size_t max_query_size, /// If (end - pos) > max_query_size and query is longer than max_query_size then throws "Max query size exceeded".
/// Disabled if zero. Is used in order to check query size if buffer can contains data for INSERT query.
bool allow_multi_statements,
size_t max_query_size,
size_t max_parser_depth,
bool skip_insignificant = true); /// If true, lexer will skip all insignificant tokens (e.g. whitespaces)
size_t max_parser_backtracks,
bool skip_insignificant = true);
/// Parse query or throw an exception with error message.
@ -39,7 +36,8 @@ ASTPtr parseKQLQueryAndMovePosition(
const std::string & description,
bool allow_multi_statements,
size_t max_query_size,
size_t max_parser_depth);
size_t max_parser_depth,
size_t max_parser_backtracks);
ASTPtr parseKQLQuery(
IParser & parser,
@ -47,18 +45,22 @@ ASTPtr parseKQLQuery(
const char * end,
const std::string & description,
size_t max_query_size,
size_t max_parser_depth);
size_t max_parser_depth,
size_t max_parser_backtracks);
ASTPtr parseKQLQuery(
IParser & parser,
const std::string & query,
const std::string & query_description,
size_t max_query_size,
size_t max_parser_depth);
size_t max_parser_depth,
size_t max_parser_backtracks);
ASTPtr parseKQLQuery(
IParser & parser,
const std::string & query,
size_t max_query_size,
size_t max_parser_depth);
size_t max_parser_depth,
size_t max_parser_backtracks);
}

2
src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp
View File

@ -11,7 +11,7 @@ using namespace DB::MySQLParser;
static inline ASTPtr tryParserQuery(IParser & parser, const String & query)
{
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0);
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0, 0);
}
TEST(ParserAlterCommand, AddAlterCommand)

2
src/Parsers/MySQL/tests/gtest_alter_parser.cpp
View File

@ -9,7 +9,7 @@ using namespace DB::MySQLParser;
static inline ASTPtr tryParserQuery(IParser & parser, const String & query)
{
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0);
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0, 0);
}
TEST(ParserAlterQuery, AlterQuery)

4
src/Parsers/MySQL/tests/gtest_column_parser.cpp
View File

@ -17,7 +17,7 @@ TEST(ParserColumn, AllNonGeneratedColumnOption)
String input = "col_01 VARCHAR(100) NOT NULL DEFAULT NULL AUTO_INCREMENT UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf8 "
"COLUMN_FORMAT FIXED STORAGE MEMORY REFERENCES tbl_name (col_01) CHECK 1";
ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0, 0);
EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->name, "VARCHAR");
EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);
@ -42,7 +42,7 @@ TEST(ParserColumn, AllGeneratedColumnOption)
String input = "col_01 VARCHAR(100) NULL UNIQUE KEY PRIMARY KEY COMMENT 'column comment' COLLATE utf8 "
"REFERENCES tbl_name (col_01) CHECK 1 GENERATED ALWAYS AS (1) STORED";
ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0, 0);
EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->name, "VARCHAR");
EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);

10
src/Parsers/MySQL/tests/gtest_constraint_parser.cpp
View File

@ -14,7 +14,7 @@ TEST(ParserConstraint, CheckConstraint)
ParserDeclareConstraint p_constraint;
String constraint_01 = "CONSTRAINT symbol_name CHECK col_01 = 1";
ASTPtr ast_constraint_01 = parseQuery(p_constraint, constraint_01.data(), constraint_01.data() + constraint_01.size(), "", 0, 0);
ASTPtr ast_constraint_01 = parseQuery(p_constraint, constraint_01.data(), constraint_01.data() + constraint_01.size(), "", 0, 0, 0);
EXPECT_EQ(ast_constraint_01->as<ASTDeclareConstraint>()->constraint_name, "symbol_name");
auto * check_expression_01 = ast_constraint_01->as<ASTDeclareConstraint>()->check_expression->as<ASTFunction>();
EXPECT_EQ(check_expression_01->name, "equals");
@ -22,7 +22,7 @@ TEST(ParserConstraint, CheckConstraint)
EXPECT_EQ(check_expression_01->arguments->children[1]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
String constraint_02 = "CONSTRAINT CHECK col_01 = 1";
ASTPtr ast_constraint_02 = parseQuery(p_constraint, constraint_02.data(), constraint_02.data() + constraint_02.size(), "", 0, 0);
ASTPtr ast_constraint_02 = parseQuery(p_constraint, constraint_02.data(), constraint_02.data() + constraint_02.size(), "", 0, 0, 0);
EXPECT_EQ(ast_constraint_02->as<ASTDeclareConstraint>()->constraint_name, "");
auto * check_expression_02 = ast_constraint_02->as<ASTDeclareConstraint>()->check_expression->as<ASTFunction>();
EXPECT_EQ(check_expression_02->name, "equals");
@ -30,7 +30,7 @@ TEST(ParserConstraint, CheckConstraint)
EXPECT_EQ(check_expression_02->arguments->children[1]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
String constraint_03 = "CHECK col_01 = 1";
ASTPtr ast_constraint_03 = parseQuery(p_constraint, constraint_03.data(), constraint_03.data() + constraint_03.size(), "", 0, 0);
ASTPtr ast_constraint_03 = parseQuery(p_constraint, constraint_03.data(), constraint_03.data() + constraint_03.size(), "", 0, 0, 0);
EXPECT_EQ(ast_constraint_03->as<ASTDeclareConstraint>()->constraint_name, "");
auto * check_expression_03 = ast_constraint_03->as<ASTDeclareConstraint>()->check_expression->as<ASTFunction>();
EXPECT_EQ(check_expression_03->name, "equals");
@ -38,7 +38,7 @@ TEST(ParserConstraint, CheckConstraint)
EXPECT_EQ(check_expression_03->arguments->children[1]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
String constraint_04 = "CONSTRAINT CHECK col_01 = 1 ENFORCED";
ASTPtr ast_constraint_04 = parseQuery(p_constraint, constraint_04.data(), constraint_04.data() + constraint_04.size(), "", 0, 0);
ASTPtr ast_constraint_04 = parseQuery(p_constraint, constraint_04.data(), constraint_04.data() + constraint_04.size(), "", 0, 0, 0);
EXPECT_TRUE(ast_constraint_04->as<ASTDeclareConstraint>()->enforced);
EXPECT_EQ(ast_constraint_04->as<ASTDeclareConstraint>()->constraint_name, "");
auto * check_expression_04 = ast_constraint_04->as<ASTDeclareConstraint>()->check_expression->as<ASTFunction>();
@ -47,7 +47,7 @@ TEST(ParserConstraint, CheckConstraint)
EXPECT_EQ(check_expression_04->arguments->children[1]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
String constraint_05 = "CONSTRAINT CHECK col_01 = 1 NOT ENFORCED";
ASTPtr ast_constraint_05 = parseQuery(p_constraint, constraint_05.data(), constraint_05.data() + constraint_05.size(), "", 0, 0);
ASTPtr ast_constraint_05 = parseQuery(p_constraint, constraint_05.data(), constraint_05.data() + constraint_05.size(), "", 0, 0, 0);
EXPECT_FALSE(ast_constraint_05->as<ASTDeclareConstraint>()->enforced);
EXPECT_EQ(ast_constraint_05->as<ASTDeclareConstraint>()->constraint_name, "");
auto * check_expression_05 = ast_constraint_05->as<ASTDeclareConstraint>()->check_expression->as<ASTFunction>();

8
src/Parsers/MySQL/tests/gtest_create_parser.cpp
View File

@ -15,16 +15,16 @@ TEST(CreateTableParser, LikeCreate)
{
ParserCreateQuery p_create_query;
String like_create_01 = "CREATE TABLE IF NOT EXISTS table_name LIKE table_name_01";
parseQuery(p_create_query, like_create_01.data(), like_create_01.data() + like_create_01.size(), "", 0, 0);
parseQuery(p_create_query, like_create_01.data(), like_create_01.data() + like_create_01.size(), "", 0, 0, 0);
String like_create_02 = "CREATE TABLE IF NOT EXISTS table_name (LIKE table_name_01)";
parseQuery(p_create_query, like_create_02.data(), like_create_02.data() + like_create_02.size(), "", 0, 0);
parseQuery(p_create_query, like_create_02.data(), like_create_02.data() + like_create_02.size(), "", 0, 0, 0);
}
TEST(CreateTableParser, SimpleCreate)
{
ParserCreateQuery p_create_query;
String input = "CREATE TABLE IF NOT EXISTS table_name(col_01 VARCHAR(100), INDEX (col_01), CHECK 1) ENGINE INNODB PARTITION BY HASH(col_01)";
ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0, 0);
EXPECT_TRUE(ast->as<ASTCreateQuery>()->if_not_exists);
EXPECT_EQ(ast->as<ASTCreateQuery>()->columns_list->as<ASTCreateDefines>()->columns->children.size(), 1);
EXPECT_EQ(ast->as<ASTCreateQuery>()->columns_list->as<ASTCreateDefines>()->indices->children.size(), 1);
@ -37,7 +37,7 @@ TEST(CreateTableParser, SS)
{
ParserCreateQuery p_create_query;
String input = "CREATE TABLE `test_table_1` (`a` int DEFAULT NULL, `b` int DEFAULT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci";
ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0, 0);
WriteBufferFromOStream buf(std::cerr, 4096);
ast->dumpTree(buf);
buf.finalize();

50
src/Parsers/MySQL/tests/gtest_index_parser.cpp
View File

@ -14,7 +14,7 @@ TEST(ParserIndex, AllIndexOptions)
String input = "INDEX (col_01, col_02(100), col_03 DESC) KEY_BLOCK_SIZE 3 USING HASH WITH PARSER parser_name COMMENT 'index comment' VISIBLE";
ParserDeclareIndex p_index;
ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareIndex * declare_index = ast->as<ASTDeclareIndex>();
EXPECT_EQ(declare_index->index_columns->children[0]->as<ASTIdentifier>()->name(), "col_01");
@ -33,7 +33,7 @@ TEST(ParserIndex, OptionalIndexOptions)
String input = "INDEX (col_01, col_02(100), col_03 DESC) USING HASH INVISIBLE KEY_BLOCK_SIZE 3";
ParserDeclareIndex p_index;
ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareIndex * declare_index = ast->as<ASTDeclareIndex>();
EXPECT_EQ(declare_index->index_columns->children[0]->as<ASTIdentifier>()->name(), "col_01");
@ -50,28 +50,28 @@ TEST(ParserIndex, OrdinaryIndex)
{
ParserDeclareIndex p_index;
String non_unique_index_01 = "KEY index_name USING HASH (col_01) INVISIBLE";
parseQuery(p_index, non_unique_index_01.data(), non_unique_index_01.data() + non_unique_index_01.size(), "", 0, 0);
parseQuery(p_index, non_unique_index_01.data(), non_unique_index_01.data() + non_unique_index_01.size(), "", 0, 0, 0);
String non_unique_index_02 = "INDEX index_name USING HASH (col_01) INVISIBLE";
parseQuery(p_index, non_unique_index_02.data(), non_unique_index_02.data() + non_unique_index_02.size(), "", 0, 0);
parseQuery(p_index, non_unique_index_02.data(), non_unique_index_02.data() + non_unique_index_02.size(), "", 0, 0, 0);
String fulltext_index_01 = "FULLTEXT index_name (col_01) INVISIBLE";
parseQuery(p_index, fulltext_index_01.data(), fulltext_index_01.data() + fulltext_index_01.size(), "", 0, 0);
parseQuery(p_index, fulltext_index_01.data(), fulltext_index_01.data() + fulltext_index_01.size(), "", 0, 0, 0);
String fulltext_index_02 = "FULLTEXT INDEX index_name (col_01) INVISIBLE";
parseQuery(p_index, fulltext_index_02.data(), fulltext_index_02.data() + fulltext_index_02.size(), "", 0, 0);
parseQuery(p_index, fulltext_index_02.data(), fulltext_index_02.data() + fulltext_index_02.size(), "", 0, 0, 0);
String fulltext_index_03 = "FULLTEXT KEY index_name (col_01) INVISIBLE";
parseQuery(p_index, fulltext_index_03.data(), fulltext_index_03.data() + fulltext_index_03.size(), "", 0, 0);
parseQuery(p_index, fulltext_index_03.data(), fulltext_index_03.data() + fulltext_index_03.size(), "", 0, 0, 0);
String spatial_index_01 = "SPATIAL index_name (col_01) INVISIBLE";
parseQuery(p_index, spatial_index_01.data(), spatial_index_01.data() + spatial_index_01.size(), "", 0, 0);
parseQuery(p_index, spatial_index_01.data(), spatial_index_01.data() + spatial_index_01.size(), "", 0, 0, 0);
String spatial_index_02 = "SPATIAL INDEX index_name (col_01) INVISIBLE";
parseQuery(p_index, spatial_index_02.data(), spatial_index_02.data() + spatial_index_02.size(), "", 0, 0);
parseQuery(p_index, spatial_index_02.data(), spatial_index_02.data() + spatial_index_02.size(), "", 0, 0, 0);
String spatial_index_03 = "SPATIAL KEY index_name (col_01) INVISIBLE";
parseQuery(p_index, spatial_index_03.data(), spatial_index_03.data() + spatial_index_03.size(), "", 0, 0);
parseQuery(p_index, spatial_index_03.data(), spatial_index_03.data() + spatial_index_03.size(), "", 0, 0, 0);
}
TEST(ParserIndex, ConstraintIndex)
@ -79,47 +79,47 @@ TEST(ParserIndex, ConstraintIndex)
ParserDeclareIndex p_index;
String primary_key_01 = "PRIMARY KEY (col_01) INVISIBLE";
parseQuery(p_index, primary_key_01.data(), primary_key_01.data() + primary_key_01.size(), "", 0, 0);
parseQuery(p_index, primary_key_01.data(), primary_key_01.data() + primary_key_01.size(), "", 0, 0, 0);
String primary_key_02 = "PRIMARY KEY USING BTREE (col_01) INVISIBLE";
parseQuery(p_index, primary_key_02.data(), primary_key_02.data() + primary_key_02.size(), "", 0, 0);
parseQuery(p_index, primary_key_02.data(), primary_key_02.data() + primary_key_02.size(), "", 0, 0, 0);
String primary_key_03 = "CONSTRAINT PRIMARY KEY USING BTREE (col_01) INVISIBLE";
parseQuery(p_index, primary_key_03.data(), primary_key_03.data() + primary_key_03.size(), "", 0, 0);
parseQuery(p_index, primary_key_03.data(), primary_key_03.data() + primary_key_03.size(), "", 0, 0, 0);
String primary_key_04 = "CONSTRAINT index_name PRIMARY KEY USING BTREE (col_01) INVISIBLE";
parseQuery(p_index, primary_key_04.data(), primary_key_04.data() + primary_key_04.size(), "", 0, 0);
parseQuery(p_index, primary_key_04.data(), primary_key_04.data() + primary_key_04.size(), "", 0, 0, 0);
String unique_key_01 = "UNIQUE (col_01) INVISIBLE";
parseQuery(p_index, unique_key_01.data(), unique_key_01.data() + unique_key_01.size(), "", 0, 0);
parseQuery(p_index, unique_key_01.data(), unique_key_01.data() + unique_key_01.size(), "", 0, 0, 0);
String unique_key_02 = "UNIQUE INDEX (col_01) INVISIBLE";
parseQuery(p_index, unique_key_02.data(), unique_key_02.data() + unique_key_02.size(), "", 0, 0);
parseQuery(p_index, unique_key_02.data(), unique_key_02.data() + unique_key_02.size(), "", 0, 0, 0);
String unique_key_03 = "UNIQUE KEY (col_01) INVISIBLE";
parseQuery(p_index, unique_key_03.data(), unique_key_03.data() + unique_key_03.size(), "", 0, 0);
parseQuery(p_index, unique_key_03.data(), unique_key_03.data() + unique_key_03.size(), "", 0, 0, 0);
String unique_key_04 = "UNIQUE KEY index_name (col_01) INVISIBLE";
parseQuery(p_index, unique_key_04.data(), unique_key_04.data() + unique_key_04.size(), "", 0, 0);
parseQuery(p_index, unique_key_04.data(), unique_key_04.data() + unique_key_04.size(), "", 0, 0, 0);
String unique_key_05 = "UNIQUE KEY index_name USING HASH (col_01) INVISIBLE";
parseQuery(p_index, unique_key_05.data(), unique_key_05.data() + unique_key_05.size(), "", 0, 0);
parseQuery(p_index, unique_key_05.data(), unique_key_05.data() + unique_key_05.size(), "", 0, 0, 0);
String unique_key_06 = "CONSTRAINT UNIQUE KEY index_name USING HASH (col_01) INVISIBLE";
parseQuery(p_index, unique_key_06.data(), unique_key_06.data() + unique_key_06.size(), "", 0, 0);
parseQuery(p_index, unique_key_06.data(), unique_key_06.data() + unique_key_06.size(), "", 0, 0, 0);
String unique_key_07 = "CONSTRAINT index_name UNIQUE KEY index_name_1 USING HASH (col_01) INVISIBLE";
parseQuery(p_index, unique_key_07.data(), unique_key_07.data() + unique_key_07.size(), "", 0, 0);
parseQuery(p_index, unique_key_07.data(), unique_key_07.data() + unique_key_07.size(), "", 0, 0, 0);
String foreign_key_01 = "FOREIGN KEY (col_01) REFERENCES tbl_name (col_01)";
parseQuery(p_index, foreign_key_01.data(), foreign_key_01.data() + foreign_key_01.size(), "", 0, 0);
parseQuery(p_index, foreign_key_01.data(), foreign_key_01.data() + foreign_key_01.size(), "", 0, 0, 0);
String foreign_key_02 = "FOREIGN KEY index_name (col_01) REFERENCES tbl_name (col_01)";
parseQuery(p_index, foreign_key_02.data(), foreign_key_02.data() + foreign_key_02.size(), "", 0, 0);
parseQuery(p_index, foreign_key_02.data(), foreign_key_02.data() + foreign_key_02.size(), "", 0, 0, 0);
String foreign_key_03 = "CONSTRAINT FOREIGN KEY index_name (col_01) REFERENCES tbl_name (col_01)";
parseQuery(p_index, foreign_key_03.data(), foreign_key_03.data() + foreign_key_03.size(), "", 0, 0);
parseQuery(p_index, foreign_key_03.data(), foreign_key_03.data() + foreign_key_03.size(), "", 0, 0, 0);
String foreign_key_04 = "CONSTRAINT index_name FOREIGN KEY index_name_01 (col_01) REFERENCES tbl_name (col_01)";
parseQuery(p_index, foreign_key_04.data(), foreign_key_04.data() + foreign_key_04.size(), "", 0, 0);
parseQuery(p_index, foreign_key_04.data(), foreign_key_04.data() + foreign_key_04.size(), "", 0, 0, 0);
}

26
src/Parsers/MySQL/tests/gtest_partition_options_parser.cpp
View File

@ -14,14 +14,14 @@ TEST(ParserPartitionOptions, HashPatitionOptions)
String hash_partition = "PARTITION BY HASH(col_01)";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast_01 = parseQuery(p_partition_options, hash_partition.data(), hash_partition.data() + hash_partition.size(), "", 0, 0);
ASTPtr ast_01 = parseQuery(p_partition_options, hash_partition.data(), hash_partition.data() + hash_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_01->partition_type, "hash");
EXPECT_EQ(declare_partition_options_01->partition_expression->as<ASTIdentifier>()->name(), "col_01");
String linear_hash_partition = "PARTITION BY LINEAR HASH(col_01)";
ASTPtr ast_02 = parseQuery(p_partition_options, linear_hash_partition.data(), linear_hash_partition.data() + linear_hash_partition.size(), "", 0, 0);
ASTPtr ast_02 = parseQuery(p_partition_options, linear_hash_partition.data(), linear_hash_partition.data() + linear_hash_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_02->partition_type, "linear_hash");
@ -33,14 +33,14 @@ TEST(ParserPartitionOptions, KeyPatitionOptions)
String key_partition = "PARTITION BY KEY(col_01)";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast_01 = parseQuery(p_partition_options, key_partition.data(), key_partition.data() + key_partition.size(), "", 0, 0);
ASTPtr ast_01 = parseQuery(p_partition_options, key_partition.data(), key_partition.data() + key_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_01->partition_type, "key");
EXPECT_EQ(declare_partition_options_01->partition_expression->as<ASTIdentifier>()->name(), "col_01");
String linear_key_partition = "PARTITION BY LINEAR KEY(col_01, col_02)";
ASTPtr ast_02 = parseQuery(p_partition_options, linear_key_partition.data(), linear_key_partition.data() + linear_key_partition.size(), "", 0, 0);
ASTPtr ast_02 = parseQuery(p_partition_options, linear_key_partition.data(), linear_key_partition.data() + linear_key_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_02->partition_type, "linear_key");
@ -49,7 +49,7 @@ TEST(ParserPartitionOptions, KeyPatitionOptions)
EXPECT_EQ(columns_list->children[1]->as<ASTIdentifier>()->name(), "col_02");
String key_partition_with_algorithm = "PARTITION BY KEY ALGORITHM=1 (col_01)";
ASTPtr ast_03 = parseQuery(p_partition_options, key_partition_with_algorithm.data(), key_partition_with_algorithm.data() + key_partition_with_algorithm.size(), "", 0, 0);
ASTPtr ast_03 = parseQuery(p_partition_options, key_partition_with_algorithm.data(), key_partition_with_algorithm.data() + key_partition_with_algorithm.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_03 = ast_03->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_03->partition_type, "key_1");
@ -61,14 +61,14 @@ TEST(ParserPartitionOptions, RangePatitionOptions)
String range_partition = "PARTITION BY RANGE(col_01)";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast_01 = parseQuery(p_partition_options, range_partition.data(), range_partition.data() + range_partition.size(), "", 0, 0);
ASTPtr ast_01 = parseQuery(p_partition_options, range_partition.data(), range_partition.data() + range_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_01->partition_type, "range");
EXPECT_EQ(declare_partition_options_01->partition_expression->as<ASTIdentifier>()->name(), "col_01");
String range_columns_partition = "PARTITION BY RANGE COLUMNS(col_01, col_02)";
ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0);
ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_02->partition_type, "range");
@ -82,14 +82,14 @@ TEST(ParserPartitionOptions, ListPatitionOptions)
String range_partition = "PARTITION BY LIST(col_01)";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast_01 = parseQuery(p_partition_options, range_partition.data(), range_partition.data() + range_partition.size(), "", 0, 0);
ASTPtr ast_01 = parseQuery(p_partition_options, range_partition.data(), range_partition.data() + range_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_01->partition_type, "list");
EXPECT_EQ(declare_partition_options_01->partition_expression->as<ASTIdentifier>()->name(), "col_01");
String range_columns_partition = "PARTITION BY LIST COLUMNS(col_01, col_02)";
ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0);
ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options_02->partition_type, "list");
@ -103,7 +103,7 @@ TEST(ParserPartitionOptions, PatitionNumberOptions)
String numbers_partition = "PARTITION BY KEY(col_01) PARTITIONS 2";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast = parseQuery(p_partition_options, numbers_partition.data(), numbers_partition.data() + numbers_partition.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_partition_options, numbers_partition.data(), numbers_partition.data() + numbers_partition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options = ast->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options->partition_type, "key");
@ -116,7 +116,7 @@ TEST(ParserPartitionOptions, PatitionWithSubpartitionOptions)
String partition_with_subpartition = "PARTITION BY KEY(col_01) PARTITIONS 3 SUBPARTITION BY HASH(col_02) SUBPARTITIONS 4";
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast = parseQuery(p_partition_options, partition_with_subpartition.data(), partition_with_subpartition.data() + partition_with_subpartition.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_partition_options, partition_with_subpartition.data(), partition_with_subpartition.data() + partition_with_subpartition.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options = ast->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options->partition_type, "key");
@ -134,7 +134,7 @@ TEST(ParserPartitionOptions, PatitionOptionsWithDeclarePartition)
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast = parseQuery(p_partition_options,
partition_options_with_declare.data(),
partition_options_with_declare.data() + partition_options_with_declare.size(), "", 0, 0);
partition_options_with_declare.data() + partition_options_with_declare.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options = ast->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options->partition_type, "key");
@ -153,7 +153,7 @@ TEST(ParserPartitionOptions, PatitionOptionsWithDeclarePartitions)
ParserDeclarePartitionOptions p_partition_options;
ASTPtr ast = parseQuery(p_partition_options,
partition_options_with_declare.data(),
partition_options_with_declare.data() + partition_options_with_declare.size(), "", 0, 0);
partition_options_with_declare.data() + partition_options_with_declare.size(), "", 0, 0, 0);
ASTDeclarePartitionOptions * declare_partition_options = ast->as<ASTDeclarePartitionOptions>();
EXPECT_EQ(declare_partition_options->partition_type, "key");

21
src/Parsers/MySQL/tests/gtest_partition_parser.cpp
View File

@ -17,7 +17,7 @@ TEST(ParserPartition, AllPatitionOptions)
" TABLESPACE table_space_name";
ParserDeclarePartition p_partition;
ASTPtr ast = parseQuery(p_partition, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_partition, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition = ast->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition->partition_name, "partition_name");
@ -35,7 +35,7 @@ TEST(ParserPartition, OptionalPatitionOptions)
{
String input = "PARTITION partition_name STORAGE engine = engine_name max_rows 1000 min_rows 0 tablespace table_space_name";
ParserDeclarePartition p_partition;
ASTPtr ast = parseQuery(p_partition, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_partition, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition = ast->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition->partition_name, "partition_name");
@ -50,7 +50,7 @@ TEST(ParserPartition, PatitionOptionsWithLessThan)
{
ParserDeclarePartition p_partition;
String partition_01 = "PARTITION partition_01 VALUES LESS THAN (1991) STORAGE engine = engine_name";
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0);
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_01 = ast_partition_01->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_01->partition_name, "partition_01");
@ -59,7 +59,7 @@ TEST(ParserPartition, PatitionOptionsWithLessThan)
EXPECT_EQ(declare_options_01->changes["engine"]->as<ASTIdentifier>()->name(), "engine_name");
String partition_02 = "PARTITION partition_02 VALUES LESS THAN MAXVALUE STORAGE engine = engine_name";
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0);
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_02 = ast_partition_02->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_02->partition_name, "partition_02");
@ -68,7 +68,7 @@ TEST(ParserPartition, PatitionOptionsWithLessThan)
EXPECT_EQ(declare_options_02->changes["engine"]->as<ASTIdentifier>()->name(), "engine_name");
String partition_03 = "PARTITION partition_03 VALUES LESS THAN (50, MAXVALUE) STORAGE engine = engine_name";
ASTPtr ast_partition_03 = parseQuery(p_partition, partition_03.data(), partition_03.data() + partition_03.size(), "", 0, 0);
ASTPtr ast_partition_03 = parseQuery(p_partition, partition_03.data(), partition_03.data() + partition_03.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_03 = ast_partition_03->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_03->partition_name, "partition_03");
@ -79,7 +79,7 @@ TEST(ParserPartition, PatitionOptionsWithLessThan)
EXPECT_EQ(declare_options_03->changes["engine"]->as<ASTIdentifier>()->name(), "engine_name");
String partition_04 = "PARTITION partition_04 VALUES LESS THAN (MAXVALUE, MAXVALUE) STORAGE engine = engine_name";
ASTPtr ast_partition_04 = parseQuery(p_partition, partition_04.data(), partition_04.data() + partition_04.size(), "", 0, 0);
ASTPtr ast_partition_04 = parseQuery(p_partition, partition_04.data(), partition_04.data() + partition_04.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_04 = ast_partition_04->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_04->partition_name, "partition_04");
@ -94,7 +94,7 @@ TEST(ParserPartition, PatitionOptionsWithInExpression)
{
ParserDeclarePartition p_partition;
String partition_01 = "PARTITION partition_01 VALUES IN (NULL, 1991, MAXVALUE) STORAGE engine = engine_name";
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0);
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_01 = ast_partition_01->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_01->partition_name, "partition_01");
@ -106,7 +106,7 @@ TEST(ParserPartition, PatitionOptionsWithInExpression)
EXPECT_EQ(declare_options_01->changes["engine"]->as<ASTIdentifier>()->name(), "engine_name");
String partition_02 = "PARTITION partition_02 VALUES IN ((NULL, 1991), (1991, NULL), (MAXVALUE, MAXVALUE)) STORAGE engine = engine_name";
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0);
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_02 = ast_partition_02->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_02->partition_name, "partition_02");
@ -132,18 +132,17 @@ TEST(ParserPartition, PatitionOptionsWithSubpartitions)
{
ParserDeclarePartition p_partition;
String partition_01 = "PARTITION partition_01 VALUES IN (NULL, 1991, MAXVALUE) STORAGE engine = engine_name (SUBPARTITION s_p01)";
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0);
ASTPtr ast_partition_01 = parseQuery(p_partition, partition_01.data(), partition_01.data() + partition_01.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_01 = ast_partition_01->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_01->partition_name, "partition_01");
EXPECT_TRUE(declare_partition_01->subpartitions->as<ASTExpressionList>()->children[0]->as<ASTDeclareSubPartition>());
String partition_02 = "PARTITION partition_02 VALUES IN (NULL, 1991, MAXVALUE) STORAGE engine = engine_name (SUBPARTITION s_p01, SUBPARTITION s_p02)";
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0);
ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0, 0);
ASTDeclarePartition * declare_partition_02 = ast_partition_02->as<ASTDeclarePartition>();
EXPECT_EQ(declare_partition_02->partition_name, "partition_02");
EXPECT_TRUE(declare_partition_02->subpartitions->as<ASTExpressionList>()->children[0]->as<ASTDeclareSubPartition>());
EXPECT_TRUE(declare_partition_02->subpartitions->as<ASTExpressionList>()->children[1]->as<ASTDeclareSubPartition>());
}

21
src/Parsers/MySQL/tests/gtest_reference_parser.cpp
View File

@ -12,12 +12,12 @@ TEST(ParserReference, SimpleReference)
ParserDeclareReference p_reference;
String reference_01 = "REFERENCES table_name (ref_col_01)";
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0);
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
String reference_02 = "REFERENCES table_name (ref_col_01, ref_col_02)";
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0);
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->reference_table_name, "table_name");
ASTPtr arguments = ast_reference_02->as<ASTDeclareReference>()->reference_expression->as<ASTFunction>()->arguments;
EXPECT_EQ(arguments->children[0]->as<ASTIdentifier>()->name(), "ref_col_01");
@ -28,19 +28,19 @@ TEST(ParserReference, ReferenceDifferenceKind)
{
ParserDeclareReference p_reference;
String reference_01 = "REFERENCES table_name (ref_col_01) MATCH FULL";
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0);
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
String reference_02 = "REFERENCES table_name (ref_col_01) MATCH PARTIAL";
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0);
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_PARTIAL);
String reference_03 = "REFERENCES table_name (ref_col_01) MATCH SIMPLE";
ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0);
ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_SIMPLE);
@ -50,7 +50,7 @@ TEST(ParserReference, ReferenceDifferenceOption)
{
ParserDeclareReference p_reference;
String reference_01 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE RESTRICT ON UPDATE RESTRICT";
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0);
ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
@ -58,7 +58,7 @@ TEST(ParserReference, ReferenceDifferenceOption)
EXPECT_EQ(ast_reference_01->as<ASTDeclareReference>()->on_update_option, ASTDeclareReference::RESTRICT);
String reference_02 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE";
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0);
ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
@ -66,7 +66,7 @@ TEST(ParserReference, ReferenceDifferenceOption)
EXPECT_EQ(ast_reference_02->as<ASTDeclareReference>()->on_update_option, ASTDeclareReference::CASCADE);
String reference_03 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE SET NULL ON UPDATE SET NULL";
ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0);
ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
@ -74,7 +74,7 @@ TEST(ParserReference, ReferenceDifferenceOption)
EXPECT_EQ(ast_reference_03->as<ASTDeclareReference>()->on_update_option, ASTDeclareReference::SET_NULL);
String reference_04 = "REFERENCES table_name (ref_col_01) MATCH FULL ON UPDATE NO ACTION ON DELETE NO ACTION";
ASTPtr ast_reference_04 = parseQuery(p_reference, reference_04.data(), reference_04.data() + reference_04.size(), "", 0, 0);
ASTPtr ast_reference_04 = parseQuery(p_reference, reference_04.data(), reference_04.data() + reference_04.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_04->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_04->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_04->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
@ -82,11 +82,10 @@ TEST(ParserReference, ReferenceDifferenceOption)
EXPECT_EQ(ast_reference_04->as<ASTDeclareReference>()->on_update_option, ASTDeclareReference::NO_ACTION);
String reference_05 = "REFERENCES table_name (ref_col_01) MATCH FULL ON UPDATE SET DEFAULT ON DELETE SET DEFAULT";
ASTPtr ast_reference_05 = parseQuery(p_reference, reference_05.data(), reference_05.data() + reference_05.size(), "", 0, 0);
ASTPtr ast_reference_05 = parseQuery(p_reference, reference_05.data(), reference_05.data() + reference_05.size(), "", 0, 0, 0);
EXPECT_EQ(ast_reference_05->as<ASTDeclareReference>()->reference_table_name, "table_name");
EXPECT_EQ(ast_reference_05->as<ASTDeclareReference>()->reference_expression->as<ASTIdentifier>()->name(), "ref_col_01");
EXPECT_EQ(ast_reference_05->as<ASTDeclareReference>()->kind, ASTDeclareReference::MATCH_FULL);
EXPECT_EQ(ast_reference_05->as<ASTDeclareReference>()->on_delete_option, ASTDeclareReference::SET_DEFAULT);
EXPECT_EQ(ast_reference_05->as<ASTDeclareReference>()->on_update_option, ASTDeclareReference::SET_DEFAULT);
}

5
src/Parsers/MySQL/tests/gtest_subpartition_parser.cpp
View File

@ -14,7 +14,7 @@ TEST(ParserSubpartition, AllSubpatitionOptions)
" DATA DIRECTORY 'data_directory' INDEX DIRECTORY 'index_directory' max_rows 1000 MIN_ROWs 0"
" TABLESPACE table_space_name";
MySQLParser::ParserDeclareSubPartition p_subpartition;
ASTPtr ast = parseQuery(p_subpartition, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_subpartition, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareSubPartition * declare_subpartition = ast->as<ASTDeclareSubPartition>();
EXPECT_EQ(declare_subpartition->logical_name, "subpartition_name");
@ -32,7 +32,7 @@ TEST(ParserSubpartition, OptionalSubpatitionOptions)
{
String input = "SUBPARTITION subpartition_name STORAGE engine = engine_name max_rows 1000 min_rows 0 tablespace table_space_name";
MySQLParser::ParserDeclareSubPartition p_subpartition;
ASTPtr ast = parseQuery(p_subpartition, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_subpartition, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareSubPartition * declare_subpartition = ast->as<ASTDeclareSubPartition>();
EXPECT_EQ(declare_subpartition->logical_name, "subpartition_name");
@ -42,4 +42,3 @@ TEST(ParserSubpartition, OptionalSubpatitionOptions)
EXPECT_EQ(declare_options->changes["max_rows"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1000);
EXPECT_EQ(declare_options->changes["tablespace"]->as<ASTIdentifier>()->name(), "table_space_name");
}

4
src/Parsers/MySQL/tests/gtest_table_options_parser.cpp
View File

@ -18,7 +18,7 @@ TEST(ParserTableOptions, AllSubpatitionOptions)
" STATS_PERSISTENT DEFAULT STATS_SAMPLE_PAGES 3 TABLESPACE tablespace_name STORAGE MEMORY UNION (table_01, table_02)";
ParserDeclareTableOptions p_table_options;
ASTPtr ast = parseQuery(p_table_options, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_table_options, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareOptions * declare_options = ast->as<ASTDeclareOptions>();
EXPECT_EQ(declare_options->changes["auto_increment"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
@ -56,7 +56,7 @@ TEST(ParserTableOptions, OptionalTableOptions)
{
String input = "STATS_AUTO_RECALC DEFAULT AUTO_INCREMENt = 1 ";
ParserDeclareTableOptions p_table_options;
ASTPtr ast = parseQuery(p_table_options, input.data(), input.data() + input.size(), "", 0, 0);
ASTPtr ast = parseQuery(p_table_options, input.data(), input.data() + input.size(), "", 0, 0, 0);
ASTDeclareOptions * declare_options = ast->as<ASTDeclareOptions>();
EXPECT_EQ(declare_options->changes["auto_increment"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);

4
src/Parsers/PRQL/ParserPRQLQuery.cpp
View File

@ -69,7 +69,9 @@ bool ParserPRQLQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
"",
false,
max_query_size,
max_parser_depth);
max_parser_depth,
max_parser_backtracks,
true);
if (!node)
throw Exception(

3
src/Parsers/PRQL/ParserPRQLQuery.h
View File

@ -13,9 +13,10 @@ private:
// These fields are not used when PRQL is disabled at build time.
[[maybe_unused]] size_t max_query_size;
[[maybe_unused]] size_t max_parser_depth;
[[maybe_unused]] size_t max_parser_backtracks;
public:
ParserPRQLQuery(size_t max_query_size_, size_t max_parser_depth_) : max_query_size{max_query_size_}, max_parser_depth{max_parser_depth_}
ParserPRQLQuery(size_t max_query_size_, size_t max_parser_depth_, size_t max_parser_backtracks_) : max_query_size(max_query_size_), max_parser_depth(max_parser_depth_), max_parser_backtracks(max_parser_backtracks_)
{
}

3
src/Parsers/ParserAlterQuery.cpp
View File

@ -865,7 +865,8 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
else if (s_modify_sql_security.ignore(pos, expected))
{
/// This is a hack so we can reuse parser from create and don't have to write `MODIFY SQL SECURITY SQL SECURITY INVOKER`
pos -= 2;
--pos;
--pos;
if (!sql_security_p.parse(pos, command_sql_security, expected))
return false;
command->type = ASTAlterCommand::MODIFY_SQL_SECURITY;

2
src/Parsers/ParserCreateQuery.h
View File

@ -286,7 +286,7 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
{
const String type_int("INT");
Tokens tokens(type_int.data(), type_int.data() + type_int.size());
Pos tmp_pos(tokens, 0);
Pos tmp_pos(tokens, pos.max_depth, pos.max_backtracks);
Expected tmp_expected;
ParserDataType().parse(tmp_pos, type, tmp_expected);
}

2
src/Parsers/QueryParameterVisitor.cpp
View File

@ -43,7 +43,7 @@ NameSet analyzeReceiveQueryParams(const std::string & query)
const char * query_end = query.data() + query.size();
ParserQuery parser(query_end);
ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "analyzeReceiveQueryParams", 0, 0);
ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "analyzeReceiveQueryParams", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
QueryParameterVisitor(query_params).visit(extract_query_ast);
NameSet query_param_names;

Some files were not shown because too many files have changed in this diff Show More