From 4083406f546ba0068096093a3dc824b51f18ce23 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 26 Oct 2020 18:49:00 +0300 Subject: [PATCH 001/491] Introduce ASTTableIdentifier --- programs/client/QueryFuzzer.cpp | 4 +- src/Interpreters/ActionsVisitor.cpp | 4 +- src/Interpreters/AddDefaultDatabaseVisitor.h | 8 +- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 6 +- .../DatabaseAndTableWithAlias.cpp | 10 +-- src/Interpreters/DatabaseAndTableWithAlias.h | 5 +- src/Interpreters/GlobalSubqueriesVisitor.h | 4 +- src/Interpreters/IdentifierSemantic.cpp | 10 --- src/Interpreters/IdentifierSemantic.h | 5 +- .../InJoinSubqueriesPreprocessor.cpp | 2 +- src/Interpreters/InterpreterInsertQuery.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Interpreters/JoinedTables.cpp | 4 +- .../MarkTableIdentifiersVisitor.cpp | 8 -- .../MarkTableIdentifiersVisitor.h | 1 - src/Interpreters/StorageID.cpp | 6 +- src/Interpreters/StorageID.h | 4 +- src/Interpreters/interpretSubquery.cpp | 2 +- src/Parsers/ASTIdentifier.cpp | 78 +++++++++---------- src/Parsers/ASTIdentifier.h | 43 ++++++---- src/Parsers/ASTSelectQuery.cpp | 2 +- src/Parsers/ASTTablesInSelectQuery.cpp | 4 +- src/Parsers/ExpressionElementParsers.cpp | 26 ++++--- src/Parsers/ExpressionElementParsers.h | 20 +++-- src/Parsers/MySQL/ASTAlterCommand.cpp | 2 +- src/Parsers/MySQL/ASTAlterQuery.cpp | 2 +- src/Parsers/MySQL/ASTCreateQuery.cpp | 2 +- src/Parsers/ParserCreateQuery.cpp | 12 +-- src/Parsers/ParserTablesInSelectQuery.cpp | 4 +- src/Storages/StorageView.cpp | 2 +- 30 files changed, 143 insertions(+), 141 deletions(-) diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp index 3fe6b8087f0..55d60e5bbda 100644 --- a/programs/client/QueryFuzzer.cpp +++ b/programs/client/QueryFuzzer.cpp @@ -272,9 +272,7 @@ void QueryFuzzer::fuzz(ASTPtr & ast) } else if (auto * table_expr = typeid_cast(ast.get())) { - fuzz(table_expr->database_and_table_name); - fuzz(table_expr->subquery); - fuzz(table_expr->table_function); + fuzz(table_expr->children); } else if (auto * expr_list = typeid_cast(ast.get())) { diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 96da40e8f6c..8ee2b3ee46b 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -668,7 +668,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & auto & child = node.arguments->children[arg]; const auto * lambda = child->as(); - const auto * identifier = child->as(); + const auto * identifier = child->as(); if (lambda && lambda->name == "lambda") { /// If the argument is a lambda expression, just remember its approximate type. @@ -714,7 +714,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } else if (identifier && (functionIsJoinGet(node.name) || functionIsDictGet(node.name)) && arg == 0) { - auto table_id = IdentifierSemantic::extractDatabaseAndTable(*identifier); + auto table_id = identifier->getTableId(); table_id = data.context.resolveStorageID(table_id, Context::ResolveOrdinary); auto column_string = ColumnString::create(); column_string->insert(table_id.getDatabaseName() + "." + table_id.getTableName()); diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h index bb684c5547a..ca577c8f087 100644 --- a/src/Interpreters/AddDefaultDatabaseVisitor.h +++ b/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -97,16 +97,16 @@ private: void visit(ASTTableExpression & table_expression, ASTPtr &) const { if (table_expression.database_and_table_name) - tryVisit(table_expression.database_and_table_name); + tryVisit(table_expression.database_and_table_name); else if (table_expression.subquery) tryVisit(table_expression.subquery); } /// @note It expects that only table (not column) identifiers are visited. - void visit(const ASTIdentifier & identifier, ASTPtr & ast) const + void visit(const ASTTableIdentifier & identifier, ASTPtr & ast) const { if (!identifier.compound()) - ast = createTableIdentifier(database_name, identifier.name()); + ast = std::make_shared(database_name, identifier.name()); } void visit(ASTSubquery & subquery, ASTPtr &) const @@ -136,7 +136,7 @@ private: { /// Second argument of the "in" function (or similar) may be a table name or a subselect. /// Rewrite the table name or descend into subselect. - if (!tryVisit(child->children[i])) + if (!tryVisit(child->children[i])) visit(child->children[i]); } else diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index 805f425beac..9c680032f82 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -50,7 +50,7 @@ void ApplyWithSubqueryVisitor::visit(ASTTableExpression & table, const Data & da { if (table.database_and_table_name) { - auto table_id = IdentifierSemantic::extractDatabaseAndTable(table.database_and_table_name->as()); + auto table_id = table.database_and_table_name->as()->getTableId(); if (table_id.database_name.empty()) { auto subquery_it = data.subqueries.find(table_id.table_name); @@ -71,9 +71,9 @@ void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data) if (checkFunctionIsInOrGlobalInOperator(func)) { auto & ast = func.arguments->children.at(1); - if (const auto * ident = ast->as()) + if (const auto * identifier = ast->as()) { - auto table_id = IdentifierSemantic::extractDatabaseAndTable(*ident); + auto table_id = identifier->getTableId(); if (table_id.database_name.empty()) { auto subquery_it = data.subqueries.find(table_id.table_name); diff --git a/src/Interpreters/DatabaseAndTableWithAlias.cpp b/src/Interpreters/DatabaseAndTableWithAlias.cpp index e99c6b78086..ae72899e8ef 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.cpp +++ b/src/Interpreters/DatabaseAndTableWithAlias.cpp @@ -18,11 +18,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database) +DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTTableIdentifier & identifier, const String & current_database) { alias = identifier.tryGetAlias(); - auto table_id = IdentifierSemantic::extractDatabaseAndTable(identifier); + auto table_id = identifier.getTableId(); std::tie(database, table, uuid) = std::tie(table_id.database_name, table_id.table_name, table_id.uuid); if (database.empty()) database = current_database; @@ -30,9 +30,9 @@ DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTIdentifier & ident DatabaseAndTableWithAlias::DatabaseAndTableWithAlias(const ASTPtr & node, const String & current_database) { - const auto * identifier = node->as(); + const auto * identifier = node->as(); if (!identifier) - throw Exception("Logical error: identifier expected", ErrorCodes::LOGICAL_ERROR); + throw Exception("Logical error: table identifier expected", ErrorCodes::LOGICAL_ERROR); *this = DatabaseAndTableWithAlias(*identifier, current_database); } @@ -92,7 +92,7 @@ std::optional getDatabaseAndTable(const ASTSelectQuer return {}; ASTPtr database_and_table_name = table_expression->database_and_table_name; - if (!database_and_table_name || !database_and_table_name->as()) + if (!database_and_table_name || !database_and_table_name->as()) return {}; return DatabaseAndTableWithAlias(database_and_table_name); diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index 07a41c12983..670a844315c 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -14,7 +14,7 @@ namespace DB { class ASTSelectQuery; -class ASTIdentifier; +class ASTTableIdentifier; struct ASTTableExpression; class Context; @@ -22,6 +22,7 @@ class Context; /// Extracts database name (and/or alias) from table expression or identifier struct DatabaseAndTableWithAlias { + // TODO(ilezhankin): replace with ASTTableIdentifier String database; String table; String alias; @@ -29,7 +30,7 @@ struct DatabaseAndTableWithAlias DatabaseAndTableWithAlias() = default; DatabaseAndTableWithAlias(const ASTPtr & identifier_node, const String & current_database = ""); - DatabaseAndTableWithAlias(const ASTIdentifier & identifier, const String & current_database = ""); + DatabaseAndTableWithAlias(const ASTTableIdentifier & identifier, const String & current_database = ""); DatabaseAndTableWithAlias(const ASTTableExpression & table_expression, const String & current_database = ""); /// "alias." or "table." if alias is empty diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 719794f0607..3c765a0e3a6 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -113,11 +113,11 @@ public: * instead of doing a subquery, you just need to read it. */ - auto database_and_table_name = createTableIdentifier("", external_table_name); + auto database_and_table_name = std::make_shared(external_table_name); if (set_alias) { String alias = subquery_or_table_name->tryGetAlias(); - if (auto * table_name = subquery_or_table_name->as()) + if (auto * table_name = subquery_or_table_name->as()) if (alias.empty()) alias = table_name->shortName(); database_and_table_name->setAlias(alias); diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index a1fc533eb7f..034930c2eec 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -144,16 +144,6 @@ std::optional IdentifierSemantic::chooseTableColumnMatch(const ASTIdenti return tryChooseTable(identifier, tables, ambiguous, true); } -StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & identifier) -{ - if (identifier.name_parts.size() > 2) - throw Exception("Syntax error: more than two components in table expression", ErrorCodes::SYNTAX_ERROR); - - if (identifier.name_parts.size() == 2) - return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid }; - return { "", identifier.name_parts[0], identifier.uuid }; -} - std::optional IdentifierSemantic::extractNestedName(const ASTIdentifier & identifier, const String & table_name) { if (identifier.name_parts.size() == 3 && table_name == identifier.name_parts[0]) diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 80b55ba0537..e0b698ec36d 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -1,9 +1,7 @@ #pragma once -#include - -#include #include +#include namespace DB { @@ -39,7 +37,6 @@ struct IdentifierSemantic /// @returns name for 'not a column' identifiers static std::optional getTableName(const ASTIdentifier & node); static std::optional getTableName(const ASTPtr & ast); - static StorageID extractDatabaseAndTable(const ASTIdentifier & identifier); static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 2e922393131..81e77af0b56 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -102,7 +102,7 @@ private: throw Exception("Distributed table should have an alias when distributed_product_mode set to local", ErrorCodes::DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED); - auto & identifier = database_and_table->as(); + auto & identifier = database_and_table->as(); renamed_tables.emplace_back(identifier.clone()); identifier.resetTable(database, table); } diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index aa8bcd74ea6..31fe01ff21c 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -98,7 +98,7 @@ Block InterpreterInsertQuery::getSampleBlock( auto names_and_types = columns.getOrdinary(); removeDuplicateColumns(names_and_types); auto table_expr = std::make_shared(); - table_expr->database_and_table_name = createTableIdentifier(table->getStorageID()); + table_expr->database_and_table_name = std::make_shared(table->getStorageID()); table_expr->children.push_back(table_expr->database_and_table_name); TablesWithColumns tables_with_columns; tables_with_columns.emplace_back(DatabaseAndTableWithAlias(*table_expr, context.getCurrentDatabase()), names_and_types); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d9821be4e4e..5d20d2ce3c3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -127,7 +127,7 @@ String InterpreterSelectQuery::generateFilterActions( tables->children.push_back(tables_elem); tables_elem->table_expression = table_expr; tables_elem->children.push_back(table_expr); - table_expr->database_and_table_name = createTableIdentifier(db_name, table_name); + table_expr->database_and_table_name = std::make_shared(db_name, table_name); table_expr->children.push_back(table_expr->database_and_table_name); /// Using separate expression analyzer to prevent any possible alias injection diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index c0511122c1e..dd2451003a2 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -48,7 +48,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query) auto & table_expr = join->table_expression->as(); if (table_expr.database_and_table_name) { - const auto & table_id = table_expr.database_and_table_name->as(); + const auto & table_id = table_expr.database_and_table_name->as(); String expr = "(select * from " + table_id.name() + ") as " + table_id.shortName(); // FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b", @@ -240,7 +240,7 @@ void JoinedTables::rewriteDistributedInAndJoins(ASTPtr & query) std::vector renamed; renamed.reserve(ast_tables.size()); for (auto & ast : ast_tables) - renamed.emplace_back(DatabaseAndTableWithAlias(*ast->as(), database)); + renamed.emplace_back(DatabaseAndTableWithAlias(ast->as(), database)); /// Change qualified column names in distributed subqueries using table aliases. RenameQualifiedIdentifiersVisitor::Data data(renamed); diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 78563059ed1..554fa7dacaa 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -22,14 +22,6 @@ void MarkTableIdentifiersMatcher::visit(ASTPtr & ast, Data & data) { if (auto * node_func = ast->as()) visit(*node_func, ast, data); - else if (auto * node_table = ast->as()) - visit(*node_table, ast, data); -} - -void MarkTableIdentifiersMatcher::visit(ASTTableExpression & table, ASTPtr &, Data &) -{ - if (table.database_and_table_name) - setIdentifierSpecial(table.database_and_table_name); } void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr &, Data & data) diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.h b/src/Interpreters/MarkTableIdentifiersVisitor.h index f882f322bcf..0d80b865e53 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.h +++ b/src/Interpreters/MarkTableIdentifiersVisitor.h @@ -24,7 +24,6 @@ public: static void visit(ASTPtr & ast, Data & data); private: - static void visit(ASTTableExpression & table, ASTPtr &, Data &); static void visit(const ASTFunction & func, ASTPtr &, Data &); }; diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index 2d6a4900dd3..96602d3728e 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -24,7 +24,7 @@ StorageID::StorageID(const ASTQueryWithTableAndOutput & query) assertNotEmpty(); } -StorageID::StorageID(const ASTIdentifier & table_identifier_node) +StorageID::StorageID(const ASTTableIdentifier & table_identifier_node) { DatabaseAndTableWithAlias database_table(table_identifier_node); database_name = database_table.database; @@ -35,9 +35,9 @@ StorageID::StorageID(const ASTIdentifier & table_identifier_node) StorageID::StorageID(const ASTPtr & node) { - if (const auto * identifier = dynamic_cast(node.get())) + if (const auto * identifier = node->as()) *this = StorageID(*identifier); - else if (const auto * simple_query = dynamic_cast(node.get())) + else if (const auto * simple_query = node->as()) *this = StorageID(*simple_query); else throw Exception("Unexpected AST", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index 9343f67fe7a..d663d55647a 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -25,7 +25,7 @@ namespace ErrorCodes static constexpr char const * TABLE_WITH_UUID_NAME_PLACEHOLDER = "_"; class ASTQueryWithTableAndOutput; -class ASTIdentifier; +class ASTTableIdentifier; class Context; struct StorageID @@ -41,7 +41,7 @@ struct StorageID } StorageID(const ASTQueryWithTableAndOutput & query); - StorageID(const ASTIdentifier & table_identifier_node); + StorageID(const ASTTableIdentifier & table_identifier_node); StorageID(const ASTPtr & node); String getDatabaseName() const; diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index cf343a4fda2..ba8b823633c 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -47,7 +47,7 @@ std::shared_ptr interpretSubquery( /// Subquery or table name. The name of the table is similar to the subquery `SELECT * FROM t`. const auto * subquery = table_expression->as(); const auto * function = table_expression->as(); - const auto * table = table_expression->as(); + const auto * table = table_expression->as(); if (!subquery && !table && !function) throw Exception("Table expression is undefined, Method: ExpressionAnalyzer::interpretSubquery." , ErrorCodes::LOGICAL_ERROR); diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index d980300a22a..7208ce27059 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -110,21 +110,6 @@ void ASTIdentifier::restoreTable() } } -void ASTIdentifier::resetTable(const String & database_name, const String & table_name) -{ - auto ast = createTableIdentifier(database_name, table_name); - auto & ident = ast->as(); - full_name.swap(ident.full_name); - name_parts.swap(ident.name_parts); - uuid = ident.uuid; -} - -void ASTIdentifier::updateTreeHashImpl(SipHash & hash_state) const -{ - hash_state.update(uuid); - IAST::updateTreeHashImpl(hash_state); -} - void ASTIdentifier::resetFullName() { full_name = name_parts[0]; @@ -132,21 +117,49 @@ void ASTIdentifier::resetFullName() full_name += '.' + name_parts[i]; } -ASTPtr createTableIdentifier(const String & database_name, const String & table_name) +ASTTableIdentifier::ASTTableIdentifier(const String & table_name) : ASTIdentifier({table_name}, true) { - assert(database_name != "_temporary_and_external_tables"); - return createTableIdentifier(StorageID(database_name, table_name)); } -ASTPtr createTableIdentifier(const StorageID & table_id) +ASTTableIdentifier::ASTTableIdentifier(const StorageID & table_id) + : ASTIdentifier( + table_id.database_name.empty() ? std::vector{table_id.table_name} + : std::vector{table_id.database_name, table_id.table_name}, + true) { - std::shared_ptr res; - if (table_id.database_name.empty()) - res = std::make_shared(std::vector{table_id.table_name}, true); - else - res = std::make_shared(std::vector{table_id.database_name, table_id.table_name}, true); - res->uuid = table_id.uuid; - return res; + uuid = table_id.uuid; +} + +ASTTableIdentifier::ASTTableIdentifier(const String & database_name, const String & table_name) + : ASTIdentifier({database_name, table_name}, true) +{ +} + +ASTPtr ASTTableIdentifier::clone() const +{ + auto ret = std::make_shared(*this); + ret->semantic = std::make_shared(*ret->semantic); + return ret; +} + +StorageID ASTTableIdentifier::getTableId() const +{ + if (name_parts.size() == 2) return {name_parts[0], name_parts[1], uuid}; + else return {{}, name_parts[0], uuid}; +} + +void ASTTableIdentifier::resetTable(const String & database_name, const String & table_name) +{ + auto identifier = std::make_shared(database_name, table_name); + full_name.swap(identifier->full_name); + name_parts.swap(identifier->name_parts); + uuid = identifier->uuid; +} + +void ASTTableIdentifier::updateTreeHashImpl(SipHash & hash_state) const +{ + hash_state.update(uuid); + IAST::updateTreeHashImpl(hash_state); } String getIdentifierName(const IAST * ast) @@ -185,17 +198,4 @@ void setIdentifierSpecial(ASTPtr & ast) id->semantic->special = true; } -StorageID getTableIdentifier(const ASTPtr & ast) -{ - if (!ast) - throw Exception("AST node is nullptr", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - const auto & identifier = dynamic_cast(*ast); - if (identifier.name_parts.size() > 2) - throw Exception("Logical error: more than two components in table expression", ErrorCodes::SYNTAX_ERROR); - - if (identifier.name_parts.size() == 2) - return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid }; - return { "", identifier.name_parts[0], identifier.uuid }; -} - } diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 59f698eab1c..2e5da227804 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -1,9 +1,9 @@ #pragma once -#include - -#include #include +#include + +#include namespace DB @@ -13,13 +13,10 @@ struct IdentifierSemantic; struct IdentifierSemanticImpl; struct StorageID; - -/// Identifier (column, table or alias) +/// Generic identifier. ASTTableIdentifier - for table identifier. class ASTIdentifier : public ASTWithAlias { public: - UUID uuid = UUIDHelpers::Nil; - explicit ASTIdentifier(const String & short_name); explicit ASTIdentifier(std::vector && name_parts, bool special = false); @@ -43,14 +40,10 @@ public: void restoreTable(); // TODO(ilezhankin): get rid of this - // FIXME: used only when it's needed to rewrite distributed table name to real remote table name. - void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this - - void updateTreeHashImpl(SipHash & hash_state) const override; - protected: String full_name; std::vector name_parts; + std::shared_ptr semantic; /// pimpl void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; @@ -58,8 +51,6 @@ protected: private: using ASTWithAlias::children; /// ASTIdentifier is child free - std::shared_ptr semantic; /// pimpl - friend struct IdentifierSemantic; friend ASTPtr createTableIdentifier(const StorageID & table_id); friend void setIdentifierSpecial(ASTPtr & ast); @@ -69,16 +60,34 @@ private: }; +class ASTTableIdentifier : public ASTIdentifier +{ + public: + explicit ASTTableIdentifier(const String & table_name); + explicit ASTTableIdentifier(const StorageID & table_id); + ASTTableIdentifier(const String & database_name, const String & table_name); + + String getID(char delim) const override { return "TableIdentifier" + (delim + name()); } + ASTPtr clone() const override; + + UUID uuid = UUIDHelpers::Nil; // FIXME(ilezhankin): make private + + StorageID getTableId() const; + + // FIXME: used only when it's needed to rewrite distributed table name to real remote table name. + void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this + + void updateTreeHashImpl(SipHash & hash_state) const override; +}; + + /// ASTIdentifier Helpers: hide casts and semantic. -ASTPtr createTableIdentifier(const String & database_name, const String & table_name); -ASTPtr createTableIdentifier(const StorageID & table_id); void setIdentifierSpecial(ASTPtr & ast); String getIdentifierName(const IAST * ast); std::optional tryGetIdentifierName(const IAST * ast); bool tryGetIdentifierNameInto(const IAST * ast, String & name); -StorageID getTableIdentifier(const ASTPtr & ast); inline String getIdentifierName(const ASTPtr & ast) { return getIdentifierName(ast.get()); } inline std::optional tryGetIdentifierName(const ASTPtr & ast) { return tryGetIdentifierName(ast.get()); } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 499761c4634..98dd061ee49 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -366,7 +366,7 @@ void ASTSelectQuery::replaceDatabaseAndTable(const StorageID & table_id) } String table_alias = getTableExpressionAlias(table_expression); - table_expression->database_and_table_name = createTableIdentifier(table_id); + table_expression->database_and_table_name = std::make_shared(table_id); if (!table_alias.empty()) table_expression->database_and_table_name->setAlias(table_alias); diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index eb3446ca1c4..37fa4237705 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -1,6 +1,6 @@ -#include -#include #include + +#include #include diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 3c45bd005a9..309e0fb477a 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -192,17 +192,25 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex ParserKeyword s_uuid("UUID"); UUID uuid = UUIDHelpers::Nil; - if (table_name_with_optional_uuid && parts.size() <= 2 && s_uuid.ignore(pos, expected)) + if (table_name_with_optional_uuid) { - ParserStringLiteral uuid_p; - ASTPtr ast_uuid; - if (!uuid_p.parse(pos, ast_uuid, expected)) - return false; - uuid = parseFromString(ast_uuid->as()->value.get()); - } + assert(parts.size() <= 2); - node = std::make_shared(std::move(parts)); - node->as()->uuid = uuid; + if (s_uuid.ignore(pos, expected)) + { + ParserStringLiteral uuid_p; + ASTPtr ast_uuid; + if (!uuid_p.parse(pos, ast_uuid, expected)) + return false; + uuid = parseFromString(ast_uuid->as()->value.get()); + } + + if (parts.size() == 1) node = std::make_shared(parts[0]); + else node = std::make_shared(parts[0], parts[1]); + node->as()->uuid = uuid; + } + else + node = std::make_shared(std::move(parts)); return true; } diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index 702d757761a..fa4db32ca41 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -54,8 +54,11 @@ protected: class ParserCompoundIdentifier : public IParserBase { public: - ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false) - : table_name_with_optional_uuid(table_name_with_optional_uuid_) {} + explicit ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false) + : table_name_with_optional_uuid(table_name_with_optional_uuid_) + { + } + protected: const char * getName() const override { return "compound identifier"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; @@ -106,7 +109,8 @@ protected: class ParserFunction : public IParserBase { public: - ParserFunction(bool allow_function_parameters_ = true) : allow_function_parameters(allow_function_parameters_) {} + explicit ParserFunction(bool allow_function_parameters_ = true) : allow_function_parameters(allow_function_parameters_) { } + protected: const char * getName() const override { return "function"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; @@ -287,8 +291,8 @@ protected: class ParserAlias : public IParserBase { public: - ParserAlias(bool allow_alias_without_as_keyword_) - : allow_alias_without_as_keyword(allow_alias_without_as_keyword_) {} + explicit ParserAlias(bool allow_alias_without_as_keyword_) : allow_alias_without_as_keyword(allow_alias_without_as_keyword_) { } + private: static const char * restricted_keywords[]; @@ -364,8 +368,10 @@ protected: class ParserFunctionWithKeyValueArguments : public IParserBase { public: - ParserFunctionWithKeyValueArguments(bool brackets_can_be_omitted_ = false) - : brackets_can_be_omitted(brackets_can_be_omitted_) {} + explicit ParserFunctionWithKeyValueArguments(bool brackets_can_be_omitted_ = false) : brackets_can_be_omitted(brackets_can_be_omitted_) + { + } + protected: const char * getName() const override { return "function with key-value arguments"; } diff --git a/src/Parsers/MySQL/ASTAlterCommand.cpp b/src/Parsers/MySQL/ASTAlterCommand.cpp index b6f2b925de0..3d89d4d6804 100644 --- a/src/Parsers/MySQL/ASTAlterCommand.cpp +++ b/src/Parsers/MySQL/ASTAlterCommand.cpp @@ -245,7 +245,7 @@ static inline bool parseRenameCommand(IParser::Pos & pos, ASTPtr & node, Expecte if (!ParserCompoundIdentifier(false).parse(pos, new_name, expected)) return false; - StorageID new_table_id = getTableIdentifier(new_name); + auto new_table_id = new_name->as()->getTableId(); alter_command->type = ASTAlterCommand::RENAME_TABLE; alter_command->new_table_name = new_table_id.table_name; alter_command->new_database_name = new_table_id.database_name; diff --git a/src/Parsers/MySQL/ASTAlterQuery.cpp b/src/Parsers/MySQL/ASTAlterQuery.cpp index 92814e42d82..a6eb85e7472 100644 --- a/src/Parsers/MySQL/ASTAlterQuery.cpp +++ b/src/Parsers/MySQL/ASTAlterQuery.cpp @@ -46,7 +46,7 @@ bool ParserAlterQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & e node = alter_query; alter_query->command_list = command_list; - StorageID table_id = getTableIdentifier(table); + auto table_id = table->as()->getTableId(); alter_query->table = table_id.table_name; alter_query->database = table_id.database_name; diff --git a/src/Parsers/MySQL/ASTCreateQuery.cpp b/src/Parsers/MySQL/ASTCreateQuery.cpp index 6c44a915f73..f45966c473e 100644 --- a/src/Parsers/MySQL/ASTCreateQuery.cpp +++ b/src/Parsers/MySQL/ASTCreateQuery.cpp @@ -101,7 +101,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) create_query->temporary = is_temporary; create_query->if_not_exists = if_not_exists; - StorageID table_id = getTableIdentifier(table); + auto table_id = table->as()->getTableId(); create_query->table = table_id.table_name; create_query->database = table_id.database_name; create_query->like_table = like_table; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 6416e08d93b..3bc8035efe9 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -410,7 +410,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return false; } - StorageID table_id = getTableIdentifier(table); + auto table_id = table->as()->getTableId(); // Shortcut for ATTACH a previously detached table if (attach && (!pos.isValid() || pos.get().type == TokenType::Semicolon)) @@ -602,14 +602,14 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e query->if_not_exists = if_not_exists; query->is_live_view = true; - StorageID table_id = getTableIdentifier(table); + auto table_id = table->as()->getTableId(); query->database = table_id.database_name; query->table = table_id.table_name; query->uuid = table_id.uuid; query->cluster = cluster_str; if (to_table) - query->to_table_id = getTableIdentifier(to_table); + query->to_table_id = to_table->as()->getTableId(); query->set(query->columns_list, columns_list); @@ -807,14 +807,14 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->is_populate = is_populate; query->replace_view = replace_view; - StorageID table_id = getTableIdentifier(table); + auto table_id = table->as()->getTableId(); query->database = table_id.database_name; query->table = table_id.table_name; query->uuid = table_id.uuid; query->cluster = cluster_str; if (to_table) - query->to_table_id = getTableIdentifier(to_table); + query->to_table_id = to_table->as()->getTableId(); query->set(query->columns_list, columns_list); query->set(query->storage, storage); @@ -892,7 +892,7 @@ bool ParserCreateDictionaryQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, E query->is_dictionary = true; query->attach = attach; - StorageID dict_id = getTableIdentifier(name); + auto dict_id = name->as()->getTableId(); query->database = dict_id.database_name; query->table = dict_id.table_name; query->uuid = dict_id.uuid; diff --git a/src/Parsers/ParserTablesInSelectQuery.cpp b/src/Parsers/ParserTablesInSelectQuery.cpp index a13baf69420..d83c3602c90 100644 --- a/src/Parsers/ParserTablesInSelectQuery.cpp +++ b/src/Parsers/ParserTablesInSelectQuery.cpp @@ -23,7 +23,7 @@ bool ParserTableExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->subquery, expected) && !ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->table_function, expected) - && !ParserWithOptionalAlias(std::make_unique(), true).parse(pos, res->database_and_table_name, expected)) + && !ParserWithOptionalAlias(std::make_unique(true), true).parse(pos, res->database_and_table_name, expected)) return false; /// FINAL @@ -57,6 +57,8 @@ bool ParserTableExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (res->sample_offset) res->children.emplace_back(res->sample_offset); + assert(res->database_and_table_name || res->table_function || res->subquery); + node = res; return true; } diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 4b7733c1cd2..7201aa7b092 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -107,7 +107,7 @@ void StorageView::replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_ { // If it's a view table function, add a fake db.table name. if (table_expression->table_function && table_expression->table_function->as()->name == "view") - table_expression->database_and_table_name = std::make_shared("__view"); + table_expression->database_and_table_name = std::make_shared("__view"); else throw Exception("Logical error: incorrect table expression", ErrorCodes::LOGICAL_ERROR); } From b74a931a89c627214d24bc72390c532a3aae8f95 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 30 Oct 2020 21:28:11 +0300 Subject: [PATCH 002/491] Fix ASTQualifiedAsterisk first child --- src/Parsers/ExpressionElementParsers.cpp | 2 +- utils/simple-backport/format-changelog.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 309e0fb477a..fcc1fa1819a 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1346,7 +1346,7 @@ bool ParserAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool ParserQualifiedAsterisk::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (!ParserCompoundIdentifier().parse(pos, node, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, node, expected)) return false; if (pos->type != TokenType::Dot) diff --git a/utils/simple-backport/format-changelog.py b/utils/simple-backport/format-changelog.py index e0fe4912d5d..49956723b35 100755 --- a/utils/simple-backport/format-changelog.py +++ b/utils/simple-backport/format-changelog.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python3.8 import argparse import collections @@ -18,7 +18,7 @@ args = parser.parse_args() def parse_one_pull_request(item): description = item['body'] # Don't skip empty lines because they delimit parts of description - lines = [line for line in [x.strip() for x in description.split('\n') if description else []]] + lines = [line for line in [x.strip() for x in description.split('\n')] if description] lines = [re.sub(r'\s+', ' ', l) for l in lines] category = '' From d69b6307b5ba42d34a92b4f977e0b8e921e56259 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 13 Nov 2020 17:13:27 +0300 Subject: [PATCH 003/491] Fix GLOBAL IN evaluation --- src/Interpreters/ActionsVisitor.cpp | 9 +++++--- src/Interpreters/GlobalSubqueriesVisitor.h | 4 +--- .../MarkTableIdentifiersVisitor.cpp | 21 ++++++++++++------- src/Parsers/ASTIdentifier.cpp | 7 +++++++ src/Parsers/ASTIdentifier.h | 7 ++++--- utils/simple-backport/format-changelog.py | 2 +- 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index f756a138a28..13c48e9dcca 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -526,6 +526,7 @@ bool ActionsMatcher::needChildVisit(const ASTPtr & node, const ASTPtr & child) { /// Visit children themself if (node->as() || + node->as() || node->as() || node->as() || node->as()) @@ -543,6 +544,8 @@ void ActionsMatcher::visit(const ASTPtr & ast, Data & data) { if (const auto * identifier = ast->as()) visit(*identifier, ast, data); + else if (const auto * table = ast->as()) + visit(*table, ast, data); else if (const auto * node = ast->as()) visit(*node, ast, data); else if (const auto * literal = ast->as()) @@ -659,9 +662,9 @@ void ActionsMatcher::visit(ASTExpressionList & expression_list, const ASTPtr &, } } -void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr & ast, Data & data) +void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr &, Data & data) { - auto column_name = ast->getColumnName(); + auto column_name = identifier.getColumnName(); if (data.hasColumn(column_name)) return; @@ -1004,7 +1007,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su const ASTPtr & right_in_operand = args.children.at(1); /// If the subquery or table name for SELECT. - const auto * identifier = right_in_operand->as(); + const auto * identifier = right_in_operand->as(); if (right_in_operand->as() || identifier) { if (no_subqueries) diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 3c765a0e3a6..4df9da14651 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -158,9 +158,7 @@ public: static bool needChildVisit(ASTPtr &, const ASTPtr & child) { /// We do not go into subqueries. - if (child->as()) - return false; - return true; + return !child->as(); } private: diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 554fa7dacaa..d77941e476d 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -24,29 +25,35 @@ void MarkTableIdentifiersMatcher::visit(ASTPtr & ast, Data & data) visit(*node_func, ast, data); } -void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr &, Data & data) +void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, Data & data) { /// `IN t` can be specified, where t is a table, which is equivalent to `IN (SELECT * FROM t)`. if (checkFunctionIsInOrGlobalInOperator(func)) { - auto & ast = func.arguments->children.at(1); + auto ast = func.arguments->children.at(1); auto opt_name = tryGetIdentifierName(ast); if (opt_name && !data.aliases.count(*opt_name)) - setIdentifierSpecial(ast); + { + ptr = func.clone(); + ptr->as()->arguments->children[1] = ast->as()->createTable(); + assert(ptr->as()->arguments->children[1]); + } } // First argument of joinGet can be a table name, perhaps with a database. // First argument of dictGet can be a dictionary name, perhaps with a database. - if (functionIsJoinGet(func.name) || functionIsDictGet(func.name)) + else if (functionIsJoinGet(func.name) || functionIsDictGet(func.name)) { if (func.arguments->children.empty()) - { return; - } auto & ast = func.arguments->children.at(0); auto opt_name = tryGetIdentifierName(ast); if (opt_name && !data.aliases.count(*opt_name)) - setIdentifierSpecial(ast); + { + ptr = func.clone(); + ptr->as()->arguments->children[0] = ast->as()->createTable(); + assert(ptr->as()->arguments->children[0]); + } } } diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 3f9fd36f348..7b4270f875d 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -142,6 +142,13 @@ void ASTIdentifier::restoreTable() } } +std::shared_ptr ASTIdentifier::createTable() const +{ + if (name_parts.size() == 1) return std::make_shared(name_parts[0]); + if (name_parts.size() == 2) return std::make_shared(name_parts[0], name_parts[1]); + return nullptr; +} + void ASTIdentifier::resetFullName() { full_name = name_parts[0]; diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 7a6ce5e3589..223fe689b80 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -14,10 +14,11 @@ struct IdentifierSemantic; struct IdentifierSemanticImpl; struct StorageID; +class ASTTableIdentifier; + /// Generic identifier. ASTTableIdentifier - for table identifier. class ASTIdentifier : public ASTWithAlias { - friend class ReplaceQueryParameterVisitor; public: explicit ASTIdentifier(const String & short_name, ASTPtr && name_param = {}); explicit ASTIdentifier(std::vector && name_parts, bool special = false, std::vector && name_params = {}); @@ -44,6 +45,7 @@ public: const String & name() const; void restoreTable(); // TODO(ilezhankin): get rid of this + std::shared_ptr createTable() const; // return |nullptr| if identifier is not table. protected: String full_name; @@ -56,10 +58,9 @@ protected: private: using ASTWithAlias::children; /// ASTIdentifier is child free + friend class ReplaceQueryParameterVisitor; friend struct IdentifierSemantic; - friend ASTPtr createTableIdentifier(const StorageID & table_id); friend void setIdentifierSpecial(ASTPtr & ast); - friend StorageID getTableIdentifier(const ASTPtr & ast); void resetFullName(); }; diff --git a/utils/simple-backport/format-changelog.py b/utils/simple-backport/format-changelog.py index 9fa42246e12..91547befed4 100755 --- a/utils/simple-backport/format-changelog.py +++ b/utils/simple-backport/format-changelog.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.8 +#!/usr/bin/python3 import argparse import collections From fb473e6f9c629bf089400bdd55e0810d1a5cc6a7 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 13 Nov 2020 17:34:47 +0300 Subject: [PATCH 004/491] More fixes --- src/Interpreters/GlobalSubqueriesVisitor.h | 4 ++-- src/Interpreters/IdentifierSemantic.cpp | 1 - src/Interpreters/MarkTableIdentifiersVisitor.cpp | 4 ++-- src/Parsers/ASTIdentifier.cpp | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 4df9da14651..7e163999d91 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -57,7 +57,7 @@ public: return; bool is_table = false; - ASTPtr subquery_or_table_name = ast; /// ASTIdentifier | ASTSubquery | ASTTableExpression + ASTPtr subquery_or_table_name = ast; /// ASTTableIdentifier | ASTSubquery | ASTTableExpression if (const auto * ast_table_expr = ast->as()) { @@ -69,7 +69,7 @@ public: is_table = true; } } - else if (ast->as()) + else if (ast->as()) is_table = true; if (!subquery_or_table_name) diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 034930c2eec..b440bb16703 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -8,7 +8,6 @@ namespace DB namespace ErrorCodes { - extern const int SYNTAX_ERROR; extern const int AMBIGUOUS_COLUMN_NAME; } diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index d77941e476d..8448b86a83d 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -32,7 +32,7 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, { auto ast = func.arguments->children.at(1); auto opt_name = tryGetIdentifierName(ast); - if (opt_name && !data.aliases.count(*opt_name)) + if (opt_name && !data.aliases.count(*opt_name) && ast->as()) { ptr = func.clone(); ptr->as()->arguments->children[1] = ast->as()->createTable(); @@ -48,7 +48,7 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, return; auto & ast = func.arguments->children.at(0); auto opt_name = tryGetIdentifierName(ast); - if (opt_name && !data.aliases.count(*opt_name)) + if (opt_name && !data.aliases.count(*opt_name) && ast->as()) { ptr = func.clone(); ptr->as()->arguments->children[0] = ast->as()->createTable(); diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 7b4270f875d..739c35a0501 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int UNEXPECTED_AST_STRUCTURE; - extern const int SYNTAX_ERROR; } ASTIdentifier::ASTIdentifier(const String & short_name, ASTPtr && name_param) @@ -221,7 +220,7 @@ bool tryGetIdentifierNameInto(const IAST * ast, String & name) { if (ast) { - if (const auto * node = ast->as()) + if (const auto * node = dynamic_cast(ast)) { name = node->name(); return true; From 4c75637ee7b3c2dead9c0f137a5320c82c0c3ae6 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 18 Nov 2020 16:20:40 +0300 Subject: [PATCH 005/491] Minor fix --- src/Interpreters/MarkTableIdentifiersVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 8448b86a83d..34b44c7f8d1 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -46,7 +46,7 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, { if (func.arguments->children.empty()) return; - auto & ast = func.arguments->children.at(0); + auto ast = func.arguments->children.at(0); auto opt_name = tryGetIdentifierName(ast); if (opt_name && !data.aliases.count(*opt_name) && ast->as()) { From 853107f0c73193f6a52a230b4cc6f8e90af6b728 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 18 Nov 2020 17:47:36 +0300 Subject: [PATCH 006/491] Revert typo --- src/Interpreters/StorageID.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index 96602d3728e..70867ce1d36 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -37,7 +37,7 @@ StorageID::StorageID(const ASTPtr & node) { if (const auto * identifier = node->as()) *this = StorageID(*identifier); - else if (const auto * simple_query = node->as()) + else if (const auto * simple_query = dynamic_cast(node.get())) *this = StorageID(*simple_query); else throw Exception("Unexpected AST", ErrorCodes::LOGICAL_ERROR); From efc4ed522d74866ae8d02811ef9cec4badf2836e Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 18 Nov 2020 17:48:24 +0300 Subject: [PATCH 007/491] Update StorageID.cpp --- src/Interpreters/StorageID.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/StorageID.cpp b/src/Interpreters/StorageID.cpp index 70867ce1d36..8cabe6dbd2c 100644 --- a/src/Interpreters/StorageID.cpp +++ b/src/Interpreters/StorageID.cpp @@ -37,7 +37,7 @@ StorageID::StorageID(const ASTPtr & node) { if (const auto * identifier = node->as()) *this = StorageID(*identifier); - else if (const auto * simple_query = dynamic_cast(node.get())) + else if (const auto * simple_query = dynamic_cast(node.get())) *this = StorageID(*simple_query); else throw Exception("Unexpected AST", ErrorCodes::LOGICAL_ERROR); From a09559b2cbbef533fa98a1a228b339c048e6a42a Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 18 Nov 2020 19:55:55 +0300 Subject: [PATCH 008/491] More fixes --- src/Interpreters/JoinToSubqueryTransformVisitor.cpp | 4 ++-- src/Interpreters/JoinedTables.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 372bbfbe648..364c38560b7 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -47,7 +47,7 @@ ASTPtr makeSubqueryTemplate() ASTPtr makeSubqueryQualifiedAsterisk() { auto asterisk = std::make_shared(); - asterisk->children.emplace_back(std::make_shared("--.s")); + asterisk->children.emplace_back(std::make_shared("--.s")); return asterisk; } @@ -115,7 +115,7 @@ private: if (child->children.size() != 1) throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR); - ASTIdentifier & identifier = child->children[0]->as(); + auto & identifier = child->children[0]->as(); data.addTableColumns(identifier.name()); } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index dd2451003a2..964879f3f20 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -110,7 +110,7 @@ private: static void visit(const ASTQualifiedAsterisk & node, const ASTPtr &, Data & data) { - ASTIdentifier & identifier = *node.children[0]->as(); + auto & identifier = node.children[0]->as(); bool rewritten = false; for (const auto & table : data) { From 2b7d0560023732f937ab9293977bf7877c440f8d Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 20 Nov 2020 16:39:56 +0300 Subject: [PATCH 009/491] Fix IN operator --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/IdentifierSemantic.cpp | 16 ---------------- src/Interpreters/IdentifierSemantic.h | 2 -- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 45230c53e81..477dcb25b6e 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -372,7 +372,7 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) if (storage()->mayBenefitFromIndexForIn(left_in_operand, context, metadata_snapshot)) { const ASTPtr & arg = args.children.at(1); - if (arg->as() || arg->as()) + if (arg->as() || arg->as()) { if (settings.use_index_for_in_with_subqueries) tryMakeSetForIndexFromSubquery(arg); diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index b440bb16703..cbf95232023 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -78,22 +78,6 @@ std::optional IdentifierSemantic::getColumnName(const ASTPtr & ast) return {}; } -std::optional IdentifierSemantic::getTableName(const ASTIdentifier & node) -{ - if (node.semantic->special) - return node.name(); - return {}; -} - -std::optional IdentifierSemantic::getTableName(const ASTPtr & ast) -{ - if (ast) - if (const auto * id = ast->as()) - if (id->semantic->special) - return id->name(); - return {}; -} - std::optional IdentifierSemantic::uncover(const ASTIdentifier & identifier) { if (identifier.semantic->covered) diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index e0b698ec36d..7ce7c018a7e 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -35,8 +35,6 @@ struct IdentifierSemantic static std::optional getColumnName(const ASTPtr & ast); /// @returns name for 'not a column' identifiers - static std::optional getTableName(const ASTIdentifier & node); - static std::optional getTableName(const ASTPtr & ast); static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); From 833c0842a1d51012fe26685937ba19cda679aac9 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 20 Nov 2020 21:38:30 +0300 Subject: [PATCH 010/491] Fix two more tests --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/TreeRewriter.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 477dcb25b6e..8e4798be8ca 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -331,7 +331,7 @@ void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr SetPtr SelectQueryExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_or_table_name) { - const auto * table = subquery_or_table_name->as(); + const auto * table = subquery_or_table_name->as(); if (!table) return nullptr; auto table_id = context.resolveStorageID(subquery_or_table_name); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index b2dbd027191..d9c5109e6a2 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -739,13 +739,13 @@ void TreeRewriter::normalize(ASTPtr & query, Aliases & aliases, const Settings & CustomizeAggregateFunctionsOrNullVisitor(data_or_null).visit(query); } - /// Creates a dictionary `aliases`: alias -> ASTPtr - QueryAliasesVisitor(aliases).visit(query); - /// Mark table ASTIdentifiers with not a column marker MarkTableIdentifiersVisitor::Data identifiers_data{aliases}; MarkTableIdentifiersVisitor(identifiers_data).visit(query); + /// Creates a dictionary `aliases`: alias -> ASTPtr + QueryAliasesVisitor(aliases).visit(query); + /// Common subexpression elimination. Rewrite rules. QueryNormalizer::Data normalizer_data(aliases, settings); QueryNormalizer(normalizer_data).visit(query); From 10747fd7c343098be5b23aef291b68c4db5c20be Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Sun, 22 Nov 2020 22:50:34 +0300 Subject: [PATCH 011/491] Better fix --- src/Interpreters/MarkTableIdentifiersVisitor.cpp | 3 +-- src/Interpreters/TreeRewriter.cpp | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 34b44c7f8d1..03787afe434 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -34,7 +34,6 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, auto opt_name = tryGetIdentifierName(ast); if (opt_name && !data.aliases.count(*opt_name) && ast->as()) { - ptr = func.clone(); ptr->as()->arguments->children[1] = ast->as()->createTable(); assert(ptr->as()->arguments->children[1]); } @@ -46,11 +45,11 @@ void MarkTableIdentifiersMatcher::visit(const ASTFunction & func, ASTPtr & ptr, { if (func.arguments->children.empty()) return; + auto ast = func.arguments->children.at(0); auto opt_name = tryGetIdentifierName(ast); if (opt_name && !data.aliases.count(*opt_name) && ast->as()) { - ptr = func.clone(); ptr->as()->arguments->children[0] = ast->as()->createTable(); assert(ptr->as()->arguments->children[0]); } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index d9c5109e6a2..b2dbd027191 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -739,13 +739,13 @@ void TreeRewriter::normalize(ASTPtr & query, Aliases & aliases, const Settings & CustomizeAggregateFunctionsOrNullVisitor(data_or_null).visit(query); } + /// Creates a dictionary `aliases`: alias -> ASTPtr + QueryAliasesVisitor(aliases).visit(query); + /// Mark table ASTIdentifiers with not a column marker MarkTableIdentifiersVisitor::Data identifiers_data{aliases}; MarkTableIdentifiersVisitor(identifiers_data).visit(query); - /// Creates a dictionary `aliases`: alias -> ASTPtr - QueryAliasesVisitor(aliases).visit(query); - /// Common subexpression elimination. Rewrite rules. QueryNormalizer::Data normalizer_data(aliases, settings); QueryNormalizer(normalizer_data).visit(query); From 8148a3bb8fc3ed46c36ebe6fba8f30441a6ea1ef Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 16 Dec 2020 16:32:23 +0300 Subject: [PATCH 012/491] [WIP] --- src/Parsers/ASTIdentifier.cpp | 6 +++++ src/Parsers/ASTIdentifier.h | 1 + src/Parsers/New/AST/AlterTableQuery.cpp | 25 +++++++++---------- src/Parsers/New/AST/AttachQuery.cpp | 6 ++--- src/Parsers/New/AST/CreateDictionaryQuery.cpp | 10 ++++---- src/Parsers/New/AST/CreateTableQuery.cpp | 10 ++++---- src/Parsers/New/AST/Identifier.cpp | 8 ++---- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 739c35a0501..2fca49cc84d 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -186,6 +186,12 @@ StorageID ASTTableIdentifier::getTableId() const else return {{}, name_parts[0], uuid}; } +String ASTTableIdentifier::getDatabaseName() const +{ + if (name_parts.size() == 2) return name_parts[0]; + else return {}; +} + void ASTTableIdentifier::resetTable(const String & database_name, const String & table_name) { auto identifier = std::make_shared(database_name, table_name); diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 0db62097b69..a29c4f3ecea 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -78,6 +78,7 @@ class ASTTableIdentifier : public ASTIdentifier UUID uuid = UUIDHelpers::Nil; // FIXME(ilezhankin): make private StorageID getTableId() const; + String getDatabaseName() const; // FIXME: used only when it's needed to rewrite distributed table name to real remote table name. void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this diff --git a/src/Parsers/New/AST/AlterTableQuery.cpp b/src/Parsers/New/AST/AlterTableQuery.cpp index 4b75e8fe1b6..6d8a08a2ec8 100644 --- a/src/Parsers/New/AST/AlterTableQuery.cpp +++ b/src/Parsers/New/AST/AlterTableQuery.cpp @@ -262,10 +262,9 @@ ASTPtr AlterTableClause::convertToOld() const if (has(FROM)) { - auto table_id = getTableIdentifier(get(FROM)->convertToOld()); - - command->from_database = table_id.database_name; - command->from_table = table_id.table_name; + auto table = get(FROM)->convertToOld(); + command->from_database = table->as()->getDatabaseName(); + command->from_table = table->as()->shortName();; command->replace = false; command->type = ASTAlterCommand::REPLACE_PARTITION; } @@ -358,9 +357,9 @@ ASTPtr AlterTableClause::convertToOld() const command->partition = get(PARTITION)->convertToOld(); command->move_destination_type = DataDestinationType::TABLE; { - auto table_id = getTableIdentifier(get(TO)->convertToOld()); - command->to_database = table_id.database_name; - command->to_table = table_id.table_name; + auto table = get(TO)->convertToOld(); + command->to_database = table->as()->getDatabaseName();; + command->to_table = table->as()->shortName(); } break; @@ -422,9 +421,9 @@ ASTPtr AlterTableClause::convertToOld() const command->replace = true; command->partition = get(PARTITION)->convertToOld(); { - auto table_id = getTableIdentifier(get(FROM)->convertToOld()); - command->from_database = table_id.database_name; - command->from_table = table_id.table_name; + auto table = get(FROM)->convertToOld(); + command->from_database = table->as()->getDatabaseName(); + command->from_table = table->as()->shortName(); } break; @@ -480,9 +479,9 @@ ASTPtr AlterTableQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = get(TABLE)->convertToOld(); + query->database = table->as()->getDatabaseName(); + query->table = table->as()->shortName(); } query->cluster = cluster_name; diff --git a/src/Parsers/New/AST/AttachQuery.cpp b/src/Parsers/New/AST/AttachQuery.cpp index 8d2e4c12346..5fba573972b 100644 --- a/src/Parsers/New/AST/AttachQuery.cpp +++ b/src/Parsers/New/AST/AttachQuery.cpp @@ -29,9 +29,9 @@ ASTPtr AttachQuery::convertToOld() const case QueryType::DICTIONARY: query->is_dictionary = true; { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = get(NAME)->convertToOld(); + query->database = table->as()->getDatabaseName(); + query->table = table->as()->shortName(); } break; } diff --git a/src/Parsers/New/AST/CreateDictionaryQuery.cpp b/src/Parsers/New/AST/CreateDictionaryQuery.cpp index 4bbf1f3d85c..75413df495b 100644 --- a/src/Parsers/New/AST/CreateDictionaryQuery.cpp +++ b/src/Parsers/New/AST/CreateDictionaryQuery.cpp @@ -239,11 +239,11 @@ ASTPtr CreateDictionaryQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid - = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table_id.uuid; + auto table = get(NAME)->convertToOld(); + query->database = table->as()->getDatabaseName(); + query->table = table->as()->shortName(); + query->uuid = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) + : table->as()->uuid; } query->cluster = cluster_name; diff --git a/src/Parsers/New/AST/CreateTableQuery.cpp b/src/Parsers/New/AST/CreateTableQuery.cpp index 77b43f525ec..31a6c73029c 100644 --- a/src/Parsers/New/AST/CreateTableQuery.cpp +++ b/src/Parsers/New/AST/CreateTableQuery.cpp @@ -108,11 +108,11 @@ ASTPtr CreateTableQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid - = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table_id.uuid; + auto table = get(NAME)->convertToOld(); + query->database = table->as()->getDatabaseName(); + query->table = table->as()->shortName(); + query->uuid = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) + : table->as()->uuid; } query->cluster = cluster_name; diff --git a/src/Parsers/New/AST/Identifier.cpp b/src/Parsers/New/AST/Identifier.cpp index a5c41bf9876..a865dbfe0bf 100644 --- a/src/Parsers/New/AST/Identifier.cpp +++ b/src/Parsers/New/AST/Identifier.cpp @@ -58,12 +58,8 @@ void TableIdentifier::makeCompound() const ASTPtr TableIdentifier::convertToOld() const { - std::vector parts; - - if (db && !db->getName().empty()) parts.push_back(db->getName()); - parts.push_back(getName()); - - return std::make_shared(std::move(parts)); + if (db) return std::make_shared(db->getName(), getName()); + else return std::make_shared(getName()); } ColumnIdentifier::ColumnIdentifier(PtrTo table_, PtrTo name) : Identifier(name->getName()), table(table_) From 4a72ad62ea8aa10ed181fb5f1410c9f2639c6d09 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 31 Dec 2020 14:52:28 +0300 Subject: [PATCH 013/491] Fix build --- .../processColumnTransformers.cpp | 2 +- src/Parsers/ASTCheckQuery.h | 1 - src/Parsers/New/AST/AlterTableQuery.cpp | 4 +- src/Parsers/New/AST/CheckQuery.cpp | 6 +-- src/Parsers/New/AST/CreateLiveViewQuery.cpp | 11 +++-- .../New/AST/CreateMaterializedViewQuery.cpp | 11 +++-- src/Parsers/New/AST/CreateTableQuery.cpp | 6 +-- src/Parsers/New/AST/CreateViewQuery.cpp | 8 ++-- src/Parsers/New/AST/ExistsQuery.cpp | 8 ++-- src/Parsers/New/AST/InsertQuery.cpp | 2 +- src/Parsers/New/AST/OptimizeQuery.cpp | 8 ++-- src/Parsers/New/AST/ShowCreateQuery.cpp | 16 +++---- src/Parsers/New/AST/SystemQuery.cpp | 42 +++++++++---------- src/Parsers/New/AST/WatchQuery.cpp | 8 ++-- 14 files changed, 65 insertions(+), 68 deletions(-) diff --git a/src/Interpreters/processColumnTransformers.cpp b/src/Interpreters/processColumnTransformers.cpp index afd99cb6f07..2a704d4a937 100644 --- a/src/Interpreters/processColumnTransformers.cpp +++ b/src/Interpreters/processColumnTransformers.cpp @@ -25,7 +25,7 @@ ASTPtr processColumnTransformers( TablesWithColumns tables_with_columns; { auto table_expr = std::make_shared(); - table_expr->database_and_table_name = createTableIdentifier(table->getStorageID()); + table_expr->database_and_table_name = std::make_shared(table->getStorageID()); table_expr->children.push_back(table_expr->database_and_table_name); tables_with_columns.emplace_back(DatabaseAndTableWithAlias(*table_expr, current_database), names_and_types); } diff --git a/src/Parsers/ASTCheckQuery.h b/src/Parsers/ASTCheckQuery.h index 0470ba4f875..fdd1179ec90 100644 --- a/src/Parsers/ASTCheckQuery.h +++ b/src/Parsers/ASTCheckQuery.h @@ -9,7 +9,6 @@ namespace DB struct ASTCheckQuery : public ASTQueryWithTableAndOutput { - ASTPtr partition; /** Get the text that identifies this element. */ diff --git a/src/Parsers/New/AST/AlterTableQuery.cpp b/src/Parsers/New/AST/AlterTableQuery.cpp index 6d8a08a2ec8..d118b45bd03 100644 --- a/src/Parsers/New/AST/AlterTableQuery.cpp +++ b/src/Parsers/New/AST/AlterTableQuery.cpp @@ -264,7 +264,7 @@ ASTPtr AlterTableClause::convertToOld() const { auto table = get(FROM)->convertToOld(); command->from_database = table->as()->getDatabaseName(); - command->from_table = table->as()->shortName();; + command->from_table = table->as()->shortName(); command->replace = false; command->type = ASTAlterCommand::REPLACE_PARTITION; } @@ -358,7 +358,7 @@ ASTPtr AlterTableClause::convertToOld() const command->move_destination_type = DataDestinationType::TABLE; { auto table = get(TO)->convertToOld(); - command->to_database = table->as()->getDatabaseName();; + command->to_database = table->as()->getDatabaseName(); command->to_table = table->as()->shortName(); } break; diff --git a/src/Parsers/New/AST/CheckQuery.cpp b/src/Parsers/New/AST/CheckQuery.cpp index 54ad8e4dac4..87a7544ec34 100644 --- a/src/Parsers/New/AST/CheckQuery.cpp +++ b/src/Parsers/New/AST/CheckQuery.cpp @@ -19,9 +19,9 @@ ASTPtr CheckQuery::convertToOld() const { auto query = std::make_shared(); - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(NAME)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); if (has(PARTITION)) query->partition = get(PARTITION)->convertToOld(); diff --git a/src/Parsers/New/AST/CreateLiveViewQuery.cpp b/src/Parsers/New/AST/CreateLiveViewQuery.cpp index b3323824924..18501884f02 100644 --- a/src/Parsers/New/AST/CreateLiveViewQuery.cpp +++ b/src/Parsers/New/AST/CreateLiveViewQuery.cpp @@ -29,18 +29,17 @@ ASTPtr CreateLiveViewQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid - = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table_id.uuid; + auto table = std::static_pointer_cast(get(NAME)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table->uuid; } if (has(TIMEOUT)) query->live_view_timeout.emplace(get(TIMEOUT)->convertToOld()->as()->value.get()); if (has(DESTINATION)) - query->to_table_id = getTableIdentifier(get(DESTINATION)->convertToOld()); + query->to_table_id = get(DESTINATION)->convertToOld()->as()->getTableId(); if (has(SCHEMA)) { diff --git a/src/Parsers/New/AST/CreateMaterializedViewQuery.cpp b/src/Parsers/New/AST/CreateMaterializedViewQuery.cpp index 24107d9dd6c..2b8a1b18b5f 100644 --- a/src/Parsers/New/AST/CreateMaterializedViewQuery.cpp +++ b/src/Parsers/New/AST/CreateMaterializedViewQuery.cpp @@ -35,15 +35,14 @@ ASTPtr CreateMaterializedViewQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid - = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table_id.uuid; + auto table = std::static_pointer_cast(get(NAME)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = has(UUID) ? parseFromString(get(UUID)->convertToOld()->as()->value.get()) : table->uuid; } if (has(DESTINATION)) - query->to_table_id = getTableIdentifier(get(DESTINATION)->convertToOld()); + query->to_table_id = get(DESTINATION)->convertToOld()->as()->getTableId(); else if (has(ENGINE)) { query->set(query->storage, get(ENGINE)->convertToOld()); diff --git a/src/Parsers/New/AST/CreateTableQuery.cpp b/src/Parsers/New/AST/CreateTableQuery.cpp index 31a6c73029c..69427812e3f 100644 --- a/src/Parsers/New/AST/CreateTableQuery.cpp +++ b/src/Parsers/New/AST/CreateTableQuery.cpp @@ -132,9 +132,9 @@ ASTPtr CreateTableQuery::convertToOld() const } case TableSchemaClause::ClauseType::TABLE: { - auto table_id = getTableIdentifier(get(SCHEMA)->convertToOld()); - query->as_database = table_id.database_name; - query->as_table = table_id.table_name; + auto table = std::static_pointer_cast(get(SCHEMA)->convertToOld()); + query->as_database = table->getDatabaseName(); + query->as_table = table->shortName(); break; } case TableSchemaClause::ClauseType::FUNCTION: diff --git a/src/Parsers/New/AST/CreateViewQuery.cpp b/src/Parsers/New/AST/CreateViewQuery.cpp index df68687eb13..b9d8031fd53 100644 --- a/src/Parsers/New/AST/CreateViewQuery.cpp +++ b/src/Parsers/New/AST/CreateViewQuery.cpp @@ -26,10 +26,10 @@ ASTPtr CreateViewQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(NAME)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(NAME)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; } query->attach = attach; diff --git a/src/Parsers/New/AST/ExistsQuery.cpp b/src/Parsers/New/AST/ExistsQuery.cpp index 0a91ea01d36..c0d35d346c4 100644 --- a/src/Parsers/New/AST/ExistsQuery.cpp +++ b/src/Parsers/New/AST/ExistsQuery.cpp @@ -31,10 +31,10 @@ ASTPtr ExistsQuery::convertToOld() const } // FIXME: this won't work if table doesn't exist - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; return query; } diff --git a/src/Parsers/New/AST/InsertQuery.cpp b/src/Parsers/New/AST/InsertQuery.cpp index 1f9325d3677..905748ba441 100644 --- a/src/Parsers/New/AST/InsertQuery.cpp +++ b/src/Parsers/New/AST/InsertQuery.cpp @@ -70,7 +70,7 @@ ASTPtr InsertQuery::convertToOld() const query->table_function = get(FUNCTION)->convertToOld(); break; case QueryType::TABLE: - query->table_id = getTableIdentifier(get(IDENTIFIER)->convertToOld()); + query->table_id = get(IDENTIFIER)->convertToOld()->as()->getTableId(); break; } diff --git a/src/Parsers/New/AST/OptimizeQuery.cpp b/src/Parsers/New/AST/OptimizeQuery.cpp index 88bb6cfbe7b..5977a2221b9 100644 --- a/src/Parsers/New/AST/OptimizeQuery.cpp +++ b/src/Parsers/New/AST/OptimizeQuery.cpp @@ -23,10 +23,10 @@ ASTPtr OptimizeQuery::convertToOld() const auto query = std::make_shared(); { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; } if (has(PARTITION)) diff --git a/src/Parsers/New/AST/ShowCreateQuery.cpp b/src/Parsers/New/AST/ShowCreateQuery.cpp index 4210f2cb67c..613b5178e62 100644 --- a/src/Parsers/New/AST/ShowCreateQuery.cpp +++ b/src/Parsers/New/AST/ShowCreateQuery.cpp @@ -47,22 +47,22 @@ ASTPtr ShowCreateQuery::convertToOld() const case QueryType::DICTIONARY: { auto query = std::make_shared(); - auto table_id = getTableIdentifier(get(IDENTIFIER)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(IDENTIFIER)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; return query; } case QueryType::TABLE: { auto query = std::make_shared(); - auto table_id = getTableIdentifier(get(IDENTIFIER)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(IDENTIFIER)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; query->temporary = temporary; return query; diff --git a/src/Parsers/New/AST/SystemQuery.cpp b/src/Parsers/New/AST/SystemQuery.cpp index a9c4b01f218..2be9ff951e0 100644 --- a/src/Parsers/New/AST/SystemQuery.cpp +++ b/src/Parsers/New/AST/SystemQuery.cpp @@ -93,25 +93,25 @@ ASTPtr SystemQuery::convertToOld() const case QueryType::DISTRIBUTED_SENDS: query->type = stop ? ASTSystemQuery::Type::STOP_DISTRIBUTED_SENDS : ASTSystemQuery::Type::START_DISTRIBUTED_SENDS; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; case QueryType::FETCHES: query->type = stop ? ASTSystemQuery::Type::STOP_FETCHES : ASTSystemQuery::Type::START_FETCHES; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; case QueryType::FLUSH_DISTRIBUTED: query->type = ASTSystemQuery::Type::FLUSH_DISTRIBUTED; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; case QueryType::FLUSH_LOGS: @@ -120,9 +120,9 @@ ASTPtr SystemQuery::convertToOld() const case QueryType::MERGES: query->type = stop ? ASTSystemQuery::Type::STOP_MERGES : ASTSystemQuery::Type::START_MERGES; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; case QueryType::RELOAD_DICTIONARIES: @@ -131,9 +131,9 @@ ASTPtr SystemQuery::convertToOld() const case QueryType::RELOAD_DICTIONARY: query->type = ASTSystemQuery::Type::RELOAD_DICTIONARY; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->target_dictionary = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->target_dictionary = table->shortName(); } break; case QueryType::REPLICATED_SENDS: @@ -142,17 +142,17 @@ ASTPtr SystemQuery::convertToOld() const case QueryType::SYNC_REPLICA: query->type = ASTSystemQuery::Type::SYNC_REPLICA; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; case QueryType::TTL_MERGES: query->type = stop ? ASTSystemQuery::Type::STOP_TTL_MERGES : ASTSystemQuery::Type::START_TTL_MERGES; { - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); } break; } diff --git a/src/Parsers/New/AST/WatchQuery.cpp b/src/Parsers/New/AST/WatchQuery.cpp index 224ff935f97..14d71007232 100644 --- a/src/Parsers/New/AST/WatchQuery.cpp +++ b/src/Parsers/New/AST/WatchQuery.cpp @@ -20,10 +20,10 @@ ASTPtr WatchQuery::convertToOld() const { auto query = std::make_shared(); - auto table_id = getTableIdentifier(get(TABLE)->convertToOld()); - query->database = table_id.database_name; - query->table = table_id.table_name; - query->uuid = table_id.uuid; + auto table = std::static_pointer_cast(get(TABLE)->convertToOld()); + query->database = table->getDatabaseName(); + query->table = table->shortName(); + query->uuid = table->uuid; query->is_watch_events = events; From b559e45d93f01b456e2d465bd9d6df76617fa52b Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 12 Apr 2021 20:19:46 +0300 Subject: [PATCH 014/491] Some minor fixes --- .../JoinToSubqueryTransformVisitor.cpp | 13 ++++--------- src/Parsers/ASTIdentifier.cpp | 1 - src/Parsers/ExpressionElementParsers.cpp | 3 ++- .../00632_aggregation_window_funnel.sql | 2 ++ .../00825_protobuf_format_no_length_delimiter.sh | 2 +- .../queries/0_stateless/01736_null_as_default.sql | 4 ++-- ...759_optimize_skip_unused_shards_zero_shards.sql | 1 + .../0_stateless/01763_max_distributed_depth.sql | 2 +- .../helpers/protobuf_length_delimited_encoder.py | 10 ++++++---- tests/queries/query_test.py | 14 ++++++++++---- tests/queries/server.py | 13 +++++++++++++ tests/queries/shell_config.sh | 2 ++ 12 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index acd31be993b..2a54dc71f84 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -467,7 +467,6 @@ std::vector normalizeColumnNamesExtractNeeded( for (ASTIdentifier * ident : identifiers) { - bool got_alias = aliases.count(ident->name()); bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias @@ -478,14 +477,10 @@ std::vector normalizeColumnNamesExtractNeeded( if (got_alias) { auto alias = aliases.find(ident->name())->second; - auto alias_table = IdentifierSemantic::getTableName(alias->ptr()); - bool alias_equals_column_name = false; - if ((!ident->isShort() && alias->ptr()->getColumnNameWithoutAlias() == ident->getColumnNameWithoutAlias()) - || (alias_table == IdentifierSemantic::getTableName(ident->ptr()) - && ident->shortName() == alias->as()->shortName())) - { - alias_equals_column_name = true; - } + bool alias_equals_column_name = alias->ptr()->getColumnNameWithoutAlias() == ident->getColumnNameWithoutAlias(); + // FIXME: check test 01600_multiple_left_joins_with_aliases + // || (alias_table == IdentifierSemantic::getTableName(ident->ptr()) + // && ident->shortName() == alias->as()->shortName())) if (!alias_equals_column_name) throw Exception("Alias clashes with qualified column '" + ident->name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); } diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index fb8274e9594..35645ee9ccd 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -79,7 +79,6 @@ void ASTIdentifier::setShortName(const String & new_name) name_parts = {new_name}; bool special = semantic->special; - //how about keep the semantic info here, such as table auto table = semantic->table; *semantic = IdentifierSemanticImpl(); diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index a0bbd56ff9e..95089028a35 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -246,7 +246,8 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex if (table_name_with_optional_uuid) { - assert(parts.size() <= 2); + if (parts.size() > 2) + return false; if (s_uuid.ignore(pos, expected)) { diff --git a/tests/queries/0_stateless/00632_aggregation_window_funnel.sql b/tests/queries/0_stateless/00632_aggregation_window_funnel.sql index d9991be5583..aa0dc804238 100644 --- a/tests/queries/0_stateless/00632_aggregation_window_funnel.sql +++ b/tests/queries/0_stateless/00632_aggregation_window_funnel.sql @@ -87,3 +87,5 @@ select 5 = windowFunnel(10000)(timestamp, event = 1000, event = 1001, event = 10 select 2 = windowFunnel(10000, 'strict_increase')(timestamp, event = 1000, event = 1001, event = 1002, event = 1003, event = 1004) from funnel_test_strict_increase; select 3 = windowFunnel(10000)(timestamp, event = 1004, event = 1004, event = 1004) from funnel_test_strict_increase; select 1 = windowFunnel(10000, 'strict_increase')(timestamp, event = 1004, event = 1004, event = 1004) from funnel_test_strict_increase; + +drop table funnel_test_strict_increase; diff --git a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh index b95d35e8256..1e8ef28a48e 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh @@ -31,7 +31,7 @@ echo "Binary representation:" hexdump -C $BINARY_FILE_PATH echo -(cd $SCHEMADIR && protoc --decode Message 00825_protobuf_format_no_length_delimiter.proto) < $BINARY_FILE_PATH +(cd $SCHEMADIR && $PROTOC_BINARY --decode Message 00825_protobuf_format_no_length_delimiter.proto) < $BINARY_FILE_PATH # Check the input in the ProtobufSingle format. echo diff --git a/tests/queries/0_stateless/01736_null_as_default.sql b/tests/queries/0_stateless/01736_null_as_default.sql index f9a4bc69acf..e3cf7649112 100644 --- a/tests/queries/0_stateless/01736_null_as_default.sql +++ b/tests/queries/0_stateless/01736_null_as_default.sql @@ -1,5 +1,5 @@ -drop table if exists test_num; +drop table if exists test_enum; create table test_enum (c Nullable(Enum16('A' = 1, 'B' = 2))) engine Log; insert into test_enum values (1), (NULL); select * from test_enum; -drop table if exists test_num; +drop table if exists test_enum; diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql index b95d640ca1a..8007b0c9b21 100644 --- a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql +++ b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql @@ -1,2 +1,3 @@ create table dist_01756 (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one', dummy); select ignore(1), * from dist_01756 where 0 settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1 +drop table dist_01756; diff --git a/tests/queries/0_stateless/01763_max_distributed_depth.sql b/tests/queries/0_stateless/01763_max_distributed_depth.sql index d1bb9e4be90..0bcb3cfe0b7 100644 --- a/tests/queries/0_stateless/01763_max_distributed_depth.sql +++ b/tests/queries/0_stateless/01763_max_distributed_depth.sql @@ -9,7 +9,7 @@ CREATE TABLE tt6 `status` String ) -ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); +ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'tt6', rand()); INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError 581 } diff --git a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py index 86c5048c8a3..893180d6cc1 100755 --- a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py +++ b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # The protobuf compiler protoc doesn't support encoding or decoding length-delimited protobuf message. -# To do that this script has been written. +# To do that this script has been written. import argparse import os.path @@ -69,7 +69,8 @@ def decode(input, output, format_schema): msg = input.read(sz) if len(msg) < sz: raise EOFError('Unexpected end of file') - with subprocess.Popen(["protoc", + protoc = os.getenv('PROTOC_BINARY', 'protoc') + with subprocess.Popen([protoc, "--decode", format_schema.message_type, format_schema.schemaname], cwd=format_schema.schemadir, stdin=subprocess.PIPE, @@ -98,7 +99,8 @@ def encode(input, output, format_schema): if line.startswith(b"MESSAGE #") or len(line) == 0: break msg += line - with subprocess.Popen(["protoc", + protoc = os.getenv('PROTOC_BINARY', 'protoc') + with subprocess.Popen([protoc, "--encode", format_schema.message_type, format_schema.schemaname], cwd=format_schema.schemadir, stdin=subprocess.PIPE, @@ -155,7 +157,7 @@ if __name__ == "__main__": group.add_argument('--decode_and_check', action='store_true', help='The same as --decode, and the utility will then encode ' ' the decoded data back to the binary form to check that the result of that encoding is the same as the input was.') args = parser.parse_args() - + custom_input_file = None custom_output_file = None try: diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index b747ac2944e..3897650e694 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -1,5 +1,3 @@ -import pytest - import difflib import os import random @@ -7,6 +5,8 @@ import string import subprocess import sys +import pytest + SKIP_LIST = [ # these couple of tests hangs everything @@ -33,7 +33,7 @@ SKIP_LIST = [ "01057_http_compression_prefer_brotli", "01080_check_for_error_incorrect_size_of_nested_column", "01083_expressions_in_engine_arguments", - # "01086_odbc_roundtrip", + "01086_odbc_roundtrip", "01088_benchmark_query_id", "01098_temporary_and_external_tables", "01099_parallel_distributed_insert_select", @@ -76,8 +76,13 @@ SKIP_LIST = [ "01599_multiline_input_and_singleline_comments", # expect-test "01601_custom_tld", "01610_client_spawn_editor", # expect-test + "01674_unicode_asan", "01676_clickhouse_client_autocomplete", # expect-test (partially) "01683_text_log_deadlock", # secure tcp + "01684_ssd_cache_dictionary_simple_key", + "01747_executable_pool_dictionary_implicit_key.sql", + "01747_join_view_filter_dictionary", + "01748_dictionary_table_dot", ] @@ -121,7 +126,8 @@ def run_shell(bin_prefix, server, database, path, reference, replace_map=None): 'CLICKHOUSE_PORT_HTTP': str(server.http_port), 'CLICKHOUSE_PORT_INTERSERVER': str(server.inter_port), 'CLICKHOUSE_TMP': server.tmp_dir, - 'CLICKHOUSE_CONFIG_CLIENT': server.client_config + 'CLICKHOUSE_CONFIG_CLIENT': server.client_config, + 'PROTOC_BINARY': os.path.abspath(os.path.join(os.path.dirname(bin_prefix), '..', 'contrib', 'protobuf', 'protoc')), # FIXME: adhoc solution } shell = subprocess.Popen([path], env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) result, error = shell.communicate() diff --git a/tests/queries/server.py b/tests/queries/server.py index ed12931e658..daa7334cb6f 100644 --- a/tests/queries/server.py +++ b/tests/queries/server.py @@ -289,6 +289,19 @@ ServerThread.DEFAULT_SERVER_CONFIG = \ + + + + + 127.0.0.1 + {tcp_port} + + + 127.0.0.2 + {tcp_port} + + + diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 5b942a95d02..4d63593f3cc 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -107,6 +107,8 @@ MYSQL_CLIENT_OPT0+=" --user ${MYSQL_CLIENT_CLICKHOUSE_USER} " export MYSQL_CLIENT_OPT="${MYSQL_CLIENT_OPT0:-} ${MYSQL_CLIENT_OPT:-}" export MYSQL_CLIENT=${MYSQL_CLIENT:="$MYSQL_CLIENT_BINARY ${MYSQL_CLIENT_OPT:-}"} +export PROTOC_BINARY=${PROTOC_BINARY:="protoc"} + function clickhouse_client_removed_host_parameter() { # removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex. From 00e85710888a5f3f6bda8adb864e5725ab655b84 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 14 Apr 2021 18:35:52 +0300 Subject: [PATCH 015/491] Fix for alter update and IN operator --- src/Interpreters/AddDefaultDatabaseVisitor.h | 6 +++++- src/Interpreters/InterpreterAlterQuery.cpp | 3 +-- src/Interpreters/MarkTableIdentifiersVisitor.cpp | 7 ++++--- src/Interpreters/MutationsInterpreter.cpp | 2 +- src/Parsers/ASTIdentifier.cpp | 3 +-- tests/queries/0_stateless/01300_wkt.sql | 2 ++ .../0_stateless/01302_polygons_distance.sql | 2 ++ ...01701_parallel_parsing_infinite_segmentation.sh | 14 ++++++++------ .../01720_country_perimeter_and_area.sh | 4 +++- ...759_optimize_skip_unused_shards_zero_shards.sql | 2 +- .../0_stateless/01760_polygon_dictionaries.sql | 1 + tests/queries/query_test.py | 4 ++++ 12 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h index c05b00d7adb..fe3edc00957 100644 --- a/src/Interpreters/AddDefaultDatabaseVisitor.h +++ b/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -102,7 +102,6 @@ private: tryVisit(table_expression.subquery); } - /// @note It expects that only table (not column) identifiers are visited. void visit(const ASTTableIdentifier & identifier, ASTPtr & ast) const { if (!identifier.compound()) @@ -134,6 +133,11 @@ private: { if (is_operator_in && i == 1) { + /// XXX: for some unknown reason this place assumes that argument can't be an alias, + /// like in the similar code in `MarkTableIdentifierVisitor`. + if (auto * identifier = child->children[i]->as()) + child->children[i] = identifier->createTable(); + /// Second argument of the "in" function (or similar) may be a table name or a subselect. /// Rewrite the table name or descend into subselect. if (!tryVisit(child->children[i])) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 9553dc87c31..6eb9970fbc7 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -65,8 +65,7 @@ BlockIO InterpreterAlterQuery::execute() auto alter_lock = table->lockForAlter(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); auto metadata_snapshot = table->getInMemoryMetadataPtr(); - /// Add default database to table identifiers that we can encounter in e.g. default expressions, - /// mutation expression, etc. + /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. AddDefaultDatabaseVisitor visitor(table_id.getDatabaseName()); ASTPtr command_list_ptr = alter.command_list->ptr(); visitor.visit(command_list_ptr); diff --git a/src/Interpreters/MarkTableIdentifiersVisitor.cpp b/src/Interpreters/MarkTableIdentifiersVisitor.cpp index 351292c5dc1..52f180aa199 100644 --- a/src/Interpreters/MarkTableIdentifiersVisitor.cpp +++ b/src/Interpreters/MarkTableIdentifiersVisitor.cpp @@ -1,12 +1,13 @@ -#include -#include -#include #include + +#include #include +#include #include #include #include + namespace DB { diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index f7872e0f742..d1ec12e0cd9 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -271,7 +271,7 @@ MutationsInterpreter::MutationsInterpreter( : storage(std::move(storage_)) , metadata_snapshot(metadata_snapshot_) , commands(std::move(commands_)) - , context(context_) + , context(Context::createCopy(context_)) , can_execute(can_execute_) , select_limits(SelectQueryOptions().analyze(!can_execute).ignoreLimits()) { diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 35645ee9ccd..4c12010dcd3 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -106,8 +106,7 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form settings.ostr << (settings.hilite ? hilite_none : ""); }; - /// It could be compound but short - if (!isShort()) + if (compound()) { for (size_t i = 0, j = 0, size = name_parts.size(); i < size; ++i) { diff --git a/tests/queries/0_stateless/01300_wkt.sql b/tests/queries/0_stateless/01300_wkt.sql index 7047bb698bb..00063d0a612 100644 --- a/tests/queries/0_stateless/01300_wkt.sql +++ b/tests/queries/0_stateless/01300_wkt.sql @@ -30,3 +30,5 @@ INSERT INTO geo VALUES ([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), INSERT INTO geo VALUES ([[[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 2); INSERT INTO geo VALUES ([[[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 3); SELECT wkt(p) FROM geo ORDER BY id; + +DROP TABLE geo; diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql index fdbd0254983..a69b5017a5f 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.sql +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -6,3 +6,5 @@ drop table if exists polygon_01302; create table polygon_01302 (x Array(Array(Array(Tuple(Float64, Float64)))), y Array(Array(Array(Tuple(Float64, Float64))))) engine=Memory(); insert into polygon_01302 values ([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); select polygonsDistanceSpherical(x, y) from polygon_01302; + +drop table polygon_01302; diff --git a/tests/queries/0_stateless/01701_parallel_parsing_infinite_segmentation.sh b/tests/queries/0_stateless/01701_parallel_parsing_infinite_segmentation.sh index d3e634eb560..edc4f6916ff 100755 --- a/tests/queries/0_stateless/01701_parallel_parsing_infinite_segmentation.sh +++ b/tests/queries/0_stateless/01701_parallel_parsing_infinite_segmentation.sh @@ -1,9 +1,11 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh ${CLICKHOUSE_CLIENT} -q "create table insert_big_json(a String, b String) engine=MergeTree() order by tuple()"; -python3 -c "[print('{{\"a\":\"{}\", \"b\":\"{}\"'.format('clickhouse'* 1000000, 'dbms' * 1000000)) for i in range(10)]; [print('{{\"a\":\"{}\", \"b\":\"{}\"}}'.format('clickhouse'* 100000, 'dbms' * 100000)) for i in range(10)]" 2>/dev/null | ${CLICKHOUSE_CLIENT} --input_format_parallel_parsing=1 --max_memory_usage=0 -q "insert into insert_big_json FORMAT JSONEachRow" 2>&1 | grep -q "min_chunk_bytes_for_parallel_parsing" && echo "Ok." || echo "FAIL" ||: \ No newline at end of file +python3 -c "[print('{{\"a\":\"{}\", \"b\":\"{}\"'.format('clickhouse'* 1000000, 'dbms' * 1000000)) for i in range(10)]; [print('{{\"a\":\"{}\", \"b\":\"{}\"}}'.format('clickhouse'* 100000, 'dbms' * 100000)) for i in range(10)]" 2>/dev/null | ${CLICKHOUSE_CLIENT} --input_format_parallel_parsing=1 --max_memory_usage=0 -q "insert into insert_big_json FORMAT JSONEachRow" 2>&1 | grep -q "min_chunk_bytes_for_parallel_parsing" && echo "Ok." || echo "FAIL" ||: + +${CLICKHOUSE_CLIENT} -q "drop table insert_big_json" diff --git a/tests/queries/0_stateless/01720_country_perimeter_and_area.sh b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh index 76dc403fb2f..d1554b4fef6 100755 --- a/tests/queries/0_stateless/01720_country_perimeter_and_area.sh +++ b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh @@ -22,4 +22,6 @@ ${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterSpherical(p) from country_ ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaSpherical(p) from country_rings" ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" -${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file +${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" + +${CLICKHOUSE_CLIENT} -q "drop table country_polygons"; diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql index 8007b0c9b21..2ddf318313f 100644 --- a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql +++ b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql @@ -1,3 +1,3 @@ create table dist_01756 (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one', dummy); -select ignore(1), * from dist_01756 where 0 settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1 +select ignore(1), * from dist_01756 where 0 settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1; drop table dist_01756; diff --git a/tests/queries/0_stateless/01760_polygon_dictionaries.sql b/tests/queries/0_stateless/01760_polygon_dictionaries.sql index 5e26d2fc306..0054be0642d 100644 --- a/tests/queries/0_stateless/01760_polygon_dictionaries.sql +++ b/tests/queries/0_stateless/01760_polygon_dictionaries.sql @@ -65,3 +65,4 @@ SELECT tuple(inf, inf) as key, dictGet('01760_db.dict_array', 'name', key); --{s DROP DICTIONARY 01760_db.dict_array; DROP TABLE 01760_db.points; DROP TABLE 01760_db.polygons; +DROP DATABASE 01760_db; diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index 3897650e694..c55c8c13f36 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -52,6 +52,7 @@ SKIP_LIST = [ "01304_direct_io", "01306_benchmark_json", "01035_lc_empty_part_bug", # FLAKY + "01175_distributed_ddl_output_mode_long", # tcp port in reference "01320_create_sync_race_condition_zookeeper", "01355_CSV_input_format_allow_errors", "01370_client_autocomplete_word_break_characters", # expect-test @@ -76,13 +77,16 @@ SKIP_LIST = [ "01599_multiline_input_and_singleline_comments", # expect-test "01601_custom_tld", "01610_client_spawn_editor", # expect-test + "01658_read_file_to_stringcolumn", "01674_unicode_asan", "01676_clickhouse_client_autocomplete", # expect-test (partially) "01683_text_log_deadlock", # secure tcp "01684_ssd_cache_dictionary_simple_key", + "01746_executable_pool_dictionary", "01747_executable_pool_dictionary_implicit_key.sql", "01747_join_view_filter_dictionary", "01748_dictionary_table_dot", + "01780_clickhouse_dictionary_source_loop", ] From fd2f5b9ede214d5714db6510f2f6073dd051b293 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 14 Apr 2021 19:36:44 +0300 Subject: [PATCH 016/491] Fix COLUMNS transformer --- src/Parsers/ExpressionElementParsers.cpp | 2 +- tests/queries/0_stateless/01785_pmj_lc_bug.sql | 3 +++ tests/queries/query_test.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 95089028a35..daa9a7f298d 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1615,7 +1615,7 @@ bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool ParserColumnsMatcher::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword columns("COLUMNS"); - ParserList columns_p(std::make_unique(true), std::make_unique(TokenType::Comma), false); + ParserList columns_p(std::make_unique(false, true), std::make_unique(TokenType::Comma), false); ParserStringLiteral regex; if (!columns.ignore(pos, expected)) diff --git a/tests/queries/0_stateless/01785_pmj_lc_bug.sql b/tests/queries/0_stateless/01785_pmj_lc_bug.sql index 722faa9b40d..3020692c80a 100644 --- a/tests/queries/0_stateless/01785_pmj_lc_bug.sql +++ b/tests/queries/0_stateless/01785_pmj_lc_bug.sql @@ -12,3 +12,6 @@ SELECT 1025 == count(n) FROM foo AS t1 ANY LEFT JOIN foo_lc AS t2 ON t1.n == t2. SELECT 1025 == count(n) FROM foo_lc AS t1 ANY LEFT JOIN foo AS t2 ON t1.n == t2.n; SELECT 1025 == count(n) FROM foo_lc AS t1 ALL LEFT JOIN foo_lc AS t2 ON t1.n == t2.n; + +DROP TABLE foo; +DROP TABLE foo_lc; diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index c55c8c13f36..30b78575ed5 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -87,6 +87,7 @@ SKIP_LIST = [ "01747_join_view_filter_dictionary", "01748_dictionary_table_dot", "01780_clickhouse_dictionary_source_loop", + "01804_dictionary_decimal256_type.sql", ] From 0288cc5a1bf1c6ffb8d41183fd5877df9b132f1a Mon Sep 17 00:00:00 2001 From: dankondr Date: Wed, 14 Apr 2021 23:29:17 +0300 Subject: [PATCH 017/491] Add dateName function --- src/Functions/dateName.cpp | 324 +++++++++++++++++++ src/Functions/registerFunctionsDateTime.cpp | 2 + src/Functions/ya.make | 1 + tests/queries/0_stateless/01811_datename.sql | 59 ++++ 4 files changed, 386 insertions(+) create mode 100644 src/Functions/dateName.cpp create mode 100644 tests/queries/0_stateless/01811_datename.sql diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp new file mode 100644 index 00000000000..7c25159775e --- /dev/null +++ b/src/Functions/dateName.cpp @@ -0,0 +1,324 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENT; +} + +namespace { + +template struct ActionValueTypeMap {}; +template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; +template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; +template <> struct ActionValueTypeMap { using ActionValueType = Int64; }; + +class FunctionDateNameImpl : public IFunction +{ +public: + static constexpr auto name = "dateName"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override + { + return name; + } + + bool useDefaultImplementationForConstants() const override { return true; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be 2 or 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + if (!WhichDataType(arguments[0].type).isString()) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1 argument of function " + getName() + + ". Must be string", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!WhichDataType(arguments[1].type).isDateOrDateTime()) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of 2 argument of function " + getName() + + "Must be a date or a date with time", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isString()) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of 3 argument of function " + getName() + + "Must be string", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + ColumnPtr executeImpl( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, [[maybe_unused]] size_t input_rows_count) const override + { + ColumnPtr res; + + if (!((res = executeType(arguments, result_type)) + || (res = executeType(arguments, result_type)) + || (res = executeType(arguments, result_type)))) + throw Exception( + "Illegal column " + arguments[1].column->getName() + " of function " + getName() + + ", must be Date or DateTime.", + ErrorCodes::ILLEGAL_COLUMN); + + return res; + } + + template + ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const + { + auto * times = checkAndGetColumn(arguments[1].column.get()); + if (!times) + return nullptr; + + const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); + if (!datepart_column) + throw Exception("Illegal column " + arguments[0].column->getName() + + " of first ('datepart') argument of function " + getName() + + ". Must be constant string.", + ErrorCodes::ILLEGAL_COLUMN); + + using T = typename ActionValueTypeMap::ActionValueType; + auto datepart_writer = DatePartWriter(); + String datepart = datepart_column->getValue(); + + if (!datepart_writer.isCorrectDatePart(datepart)) + throw Exception("Illegal value " + datepart + + " of first ('format') argument of function " + getName() + + ". Check documentation.", + ErrorCodes::BAD_ARGUMENT); + + const DateLUTImpl * time_zone_tmp; + if (std::is_same_v || std::is_same_v) + time_zone_tmp = &extractTimeZoneFromFunctionArguments(arguments, 2, 1); + else + time_zone_tmp = &DateLUT::instance(); + + const auto & vec = times->getData(); + const DateLUTImpl & time_zone = *time_zone_tmp; + + UInt32 scale [[maybe_unused]] = 0; + if constexpr (std::is_same_v) + { + scale = vec.getScale(); + } + + auto col_res = ColumnString::create(); + auto & dst_data = col_res->getChars(); + auto & dst_offsets = col_res->getOffsets(); + dst_data.resize(vec.size() * (9 /* longest possible word 'Wednesday' */ + 1 /* zero terminator */)); + dst_offsets.resize(vec.size()); + + auto * begin = reinterpret_cast(dst_data.data()); + auto * pos = begin; + + for (size_t i = 0; i < vec.size(); ++i) + { + if constexpr (std::is_same_v) + { + // since right now LUT does not support Int64-values and not format instructions for subsecond parts, + // treat DatTime64 values just as DateTime values by ignoring fractional and casting to UInt32. + const auto c = DecimalUtils::split(vec[i], scale); + datepart_writer.writeDatePart(pos, datepart, static_cast(c.whole), time_zone); + } + else + { + datepart_writer.writeDatePart(pos, datepart, vec[i], time_zone); + } + dst_offsets[i] = pos - begin; + } + dst_data.resize(pos - begin); + return col_res; + } + +private: + template + class DatePartWriter + { + public: + void writeDatePart(char *& target, const String & datepart, Time source, const DateLUTImpl & timezone) + { + datepart_functions.at(datepart)(target, source, timezone); + } + + bool isCorrectDatePart(const String &datepart) + { + return datepart_functions.find(datepart) != datepart_functions.end(); + } + + private: + const std::unordered_map datepart_functions = { + {"year", writeYear}, + {"quarter", writeQuarter}, + {"month", writeMonth}, + {"dayofyear", writeDayOfYear}, + {"day", writeDay}, + {"week", writeWeek}, + {"weekday", writeWeekday}, + {"hour", writeHour}, + {"minute", writeMinute}, + {"second", writeSecond}, + }; + + static inline void writeYear(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToYearImpl::execute(source, timezone)); + } + + static inline void writeQuarter(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToQuarterImpl::execute(source, timezone)); + } + + static inline void writeMonth(char *& target, Time source, const DateLUTImpl & timezone) + { + const auto month = ToMonthImpl::execute(source, timezone); + const String monthnames[12] = { + "January", "February", "March", + "April", "May", "June", + "July", "August", "September", + "October", "November", "December" + }; + writeString(target, monthnames[month - 1]); + } + + static inline void writeDayOfYear(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToDayOfYearImpl::execute(source, timezone)); + } + + static inline void writeDay(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToDayOfMonthImpl::execute(source, timezone)); + } + + static inline void writeWeek(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToISOWeekImpl::execute(source, timezone)); + } + + static inline void writeWeekday(char *& target, Time source, const DateLUTImpl & timezone) + { + const auto day = ToDayOfWeekImpl::execute(source, timezone); + const String daynames[12] = { + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday" + }; + writeString(target, daynames[day - 1]); + } + + static inline void writeHour(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToHourImpl::execute(source, timezone)); + } + + static inline void writeMinute(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToMinuteImpl::execute(source, timezone)); + } + + static inline void writeSecond(char *& target, Time source, const DateLUTImpl & timezone) + { + writeNumber(target, ToSecondImpl::execute(source, timezone)); + } + + static inline void writeString(char *& target, const String & value) + { + size_t size = value.size() + 1; /// With zero terminator + memcpy(target, value.data(), size); + target += size; + } + + template + static inline void writeNumber(char *& target, T value) + { + if (value < 10) + { + *target = value + '0'; + target += 2; + *target++ = 0; + } + else if (value < 100) + { + writeNumber2(target, value); + target += 3; + *target++ = 0; + } + else if (value < 1000) + { + writeNumber3(target, value); + target += 4; + *target++ = 0; + } + else if (value < 10000) + { + writeNumber4(target, value); + target += 5; + *target++ = 0; + } + else + { + throw Exception("Illegal value of second ('datetime') argument of function datePart. Check documentation.", + ErrorCodes::BAD_ARGUMENT); + } + } + + template + static inline void writeNumber2(char * p, T v) + { + memcpy(p, &digits100[v * 2], 2); + } + + template + static inline void writeNumber3(char * p, T v) + { + writeNumber2(p, v / 10); + p[2] += v % 10; + } + + template + static inline void writeNumber4(char * p, T v) + { + writeNumber2(p, v / 100); + writeNumber2(p + 2, v % 100); + } + }; +}; +} + +void registerFunctionDateName(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} diff --git a/src/Functions/registerFunctionsDateTime.cpp b/src/Functions/registerFunctionsDateTime.cpp index 441f28bfb54..abbc52c8360 100644 --- a/src/Functions/registerFunctionsDateTime.cpp +++ b/src/Functions/registerFunctionsDateTime.cpp @@ -64,6 +64,7 @@ void registerFunctionSubtractMonths(FunctionFactory &); void registerFunctionSubtractQuarters(FunctionFactory &); void registerFunctionSubtractYears(FunctionFactory &); void registerFunctionDateDiff(FunctionFactory &); +void registerFunctionDateName(FunctionFactory &); void registerFunctionToTimeZone(FunctionFactory &); void registerFunctionFormatDateTime(FunctionFactory &); void registerFunctionFromModifiedJulianDay(FunctionFactory &); @@ -134,6 +135,7 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionSubtractQuarters(factory); registerFunctionSubtractYears(factory); registerFunctionDateDiff(factory); + registerFunctionDateName(factory); registerFunctionToTimeZone(factory); registerFunctionFormatDateTime(factory); registerFunctionFromModifiedJulianDay(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 52ed54ec64f..486629e7ee9 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -222,6 +222,7 @@ SRCS( currentDatabase.cpp currentUser.cpp dateDiff.cpp + dateName.cpp date_trunc.cpp decodeXMLComponent.cpp decrypt.cpp diff --git a/tests/queries/0_stateless/01811_datename.sql b/tests/queries/0_stateless/01811_datename.sql new file mode 100644 index 00000000000..87ece62f78a --- /dev/null +++ b/tests/queries/0_stateless/01811_datename.sql @@ -0,0 +1,59 @@ +SELECT dateName('year', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('quarter', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('month', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('dayofyear', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('day', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('week', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('weekday', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('hour', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('minute', toDateTime('2021-04-14 11:22:33')); +SELECT dateName('second', toDateTime('2021-04-14 11:22:33')); + + +SELECT dateName('year', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('quarter', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('month', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('dayofyear', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('day', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('week', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('weekday', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('hour', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('minute', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('second', toDateTime64('2021-04-14 11:22:33')); + + +SELECT dateName('year', toDate('2021-04-14')); +SELECT dateName('quarter', toDate('2021-04-14')); +SELECT dateName('month', toDate('2021-04-14')); +SELECT dateName('dayofyear', toDate('2021-04-14')); +SELECT dateName('day', toDate('2021-04-14')); +SELECT dateName('week', toDate('2021-04-14')); +SELECT dateName('weekday', toDate('2021-04-14')); +SELECT dateName('hour', toDate('2021-04-14')); +SELECT dateName('minute', toDate('2021-04-14')); +SELECT dateName('second', toDate('2021-04-14')); + + +SELECT dateName('day', toDateTime('2021-04-14 11:22:33'), 'Europe/Moscow'), + dateName('day', toDateTime('2021-04-14 11:22:33'), 'UTC'); + +SELECT dateName('day', toDate('2021-04-12')); +SELECT dateName('day', toDate('2021-04-13')); +SELECT dateName('day', toDate('2021-04-14')); +SELECT dateName('day', toDate('2021-04-15')); +SELECT dateName('day', toDate('2021-04-16')); +SELECT dateName('day', toDate('2021-04-17')); +SELECT dateName('day', toDate('2021-04-18')); + +SELECT dateName('month', toDate('2021-01-14')); +SELECT dateName('month', toDate('2021-02-14')); +SELECT dateName('month', toDate('2021-03-14')); +SELECT dateName('month', toDate('2021-04-14')); +SELECT dateName('month', toDate('2021-05-14')); +SELECT dateName('month', toDate('2021-06-14')); +SELECT dateName('month', toDate('2021-07-14')); +SELECT dateName('month', toDate('2021-08-14')); +SELECT dateName('month', toDate('2021-09-14')); +SELECT dateName('month', toDate('2021-10-14')); +SELECT dateName('month', toDate('2021-11-14')); +SELECT dateName('month', toDate('2021-12-14')); From 06bc435b06184a3fa827fa46e877a6ed660ee1f0 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 15 Apr 2021 10:33:15 +0300 Subject: [PATCH 018/491] Fix code with params --- .../ReplaceQueryParameterVisitor.cpp | 19 ++++++++++--------- src/Parsers/ASTIdentifier.cpp | 13 +++++++------ src/Parsers/ASTIdentifier.h | 9 ++++++--- src/Parsers/ExpressionElementParsers.cpp | 4 ++-- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index 8d737f27e64..25051f68901 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -27,7 +27,7 @@ void ReplaceQueryParameterVisitor::visit(ASTPtr & ast) { if (ast->as()) visitQueryParameter(ast); - else if (ast->as()) + else if (ast->as() || ast->as()) visitIdentifier(ast); else visitChildren(ast); @@ -77,25 +77,26 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) void ReplaceQueryParameterVisitor::visitIdentifier(ASTPtr & ast) { - auto & ast_identifier = ast->as(); - if (ast_identifier.children.empty()) + auto ast_identifier = dynamic_pointer_cast(ast); + if (ast_identifier->children.empty()) return; - auto & name_parts = ast_identifier.name_parts; + auto & name_parts = ast_identifier->name_parts; for (size_t i = 0, j = 0, size = name_parts.size(); i < size; ++i) { if (name_parts[i].empty()) { - const auto & ast_param = ast_identifier.children[j++]->as(); + const auto & ast_param = ast_identifier->children[j++]->as(); name_parts[i] = getParamValue(ast_param.name); } } - if (!ast_identifier.semantic->special && name_parts.size() >= 2) - ast_identifier.semantic->table = ast_identifier.name_parts.end()[-2]; + /// FIXME: what should this mean? + if (!ast_identifier->semantic->special && name_parts.size() >= 2) + ast_identifier->semantic->table = ast_identifier->name_parts.end()[-2]; - ast_identifier.resetFullName(); - ast_identifier.children.clear(); + ast_identifier->resetFullName(); + ast_identifier->children.clear(); } } diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 4c12010dcd3..1cddc354a60 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -18,7 +18,7 @@ namespace ErrorCodes ASTIdentifier::ASTIdentifier(const String & short_name, ASTPtr && name_param) : full_name(short_name), name_parts{short_name}, semantic(std::make_shared()) { - if (name_param == nullptr) + if (!name_param) assert(!full_name.empty()); else children.push_back(std::move(name_param)); @@ -157,21 +157,22 @@ void ASTIdentifier::resetFullName() full_name += '.' + name_parts[i]; } -ASTTableIdentifier::ASTTableIdentifier(const String & table_name) : ASTIdentifier({table_name}, true) +ASTTableIdentifier::ASTTableIdentifier(const String & table_name, std::vector && name_params) + : ASTIdentifier({table_name}, true, std::move(name_params)) { } -ASTTableIdentifier::ASTTableIdentifier(const StorageID & table_id) +ASTTableIdentifier::ASTTableIdentifier(const StorageID & table_id, std::vector && name_params) : ASTIdentifier( table_id.database_name.empty() ? std::vector{table_id.table_name} : std::vector{table_id.database_name, table_id.table_name}, - true) + true, std::move(name_params)) { uuid = table_id.uuid; } -ASTTableIdentifier::ASTTableIdentifier(const String & database_name, const String & table_name) - : ASTIdentifier({database_name, table_name}, true) +ASTTableIdentifier::ASTTableIdentifier(const String & database_name, const String & table_name, std::vector && name_params) + : ASTIdentifier({database_name, table_name}, true, std::move(name_params)) { } diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index a29c4f3ecea..7df31244f35 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -16,6 +16,9 @@ struct StorageID; class ASTTableIdentifier; +/// FIXME: rewrite code about params - they should be substituted at the parsing stage, +/// or parsed as a separate AST entity. + /// Generic identifier. ASTTableIdentifier - for table identifier. class ASTIdentifier : public ASTWithAlias { @@ -68,9 +71,9 @@ private: class ASTTableIdentifier : public ASTIdentifier { public: - explicit ASTTableIdentifier(const String & table_name); - explicit ASTTableIdentifier(const StorageID & table_id); - ASTTableIdentifier(const String & database_name, const String & table_name); + explicit ASTTableIdentifier(const String & table_name, std::vector && name_params = {}); + explicit ASTTableIdentifier(const StorageID & table_id, std::vector && name_params = {}); + ASTTableIdentifier(const String & database_name, const String & table_name, std::vector && name_params = {}); String getID(char delim) const override { return "TableIdentifier" + (delim + name()); } ASTPtr clone() const override; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index daa9a7f298d..67c806cfa03 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -258,8 +258,8 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex uuid = parseFromString(ast_uuid->as()->value.get()); } - if (parts.size() == 1) node = std::make_shared(parts[0]); - else node = std::make_shared(parts[0], parts[1]); + if (parts.size() == 1) node = std::make_shared(parts[0], std::move(params)); + else node = std::make_shared(parts[0], parts[1], std::move(params)); node->as()->uuid = uuid; } else From e6716779b2ec28d9d3b61e27e0ecf1d8f892bb38 Mon Sep 17 00:00:00 2001 From: dankondr Date: Thu, 15 Apr 2021 20:08:55 +0300 Subject: [PATCH 019/491] Fix BAD_ARGUMENTS error code --- src/Functions/dateName.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 7c25159775e..725954b6874 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -27,7 +27,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int BAD_ARGUMENT; + extern const int BAD_ARGUMENTS; } namespace { @@ -118,7 +118,7 @@ public: throw Exception("Illegal value " + datepart + " of first ('format') argument of function " + getName() + ". Check documentation.", - ErrorCodes::BAD_ARGUMENT); + ErrorCodes::BAD_ARGUMENTS); const DateLUTImpl * time_zone_tmp; if (std::is_same_v || std::is_same_v) @@ -290,7 +290,7 @@ private: else { throw Exception("Illegal value of second ('datetime') argument of function datePart. Check documentation.", - ErrorCodes::BAD_ARGUMENT); + ErrorCodes::BAD_ARGUMENTS); } } From af48735e86d8d5dbbcfbefe25147dabc75a47712 Mon Sep 17 00:00:00 2001 From: dankondr Date: Thu, 15 Apr 2021 20:34:21 +0300 Subject: [PATCH 020/491] Fix argument check --- src/Functions/dateName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 725954b6874..4e42e9705c3 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -73,7 +73,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isString()) throw Exception( - "Illegal type " + arguments[1].type->getName() + " of 3 argument of function " + getName() + "Illegal type " + arguments[2].type->getName() + " of 3 argument of function " + getName() + "Must be string", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From 71137fd19b0a9f26b0d44a7f95f1be9473c4d48b Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 19:56:39 +0300 Subject: [PATCH 021/491] Override getArgumentsThatAreAlwaysConstant --- src/Functions/dateName.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 4e42e9705c3..5b9e984df96 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -51,6 +51,8 @@ public: bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } From bce42dc973dff1284d9771b74c9e34f336ed500f Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:10:48 +0300 Subject: [PATCH 022/491] Rewrite exception throws to fmt --- src/Functions/dateName.cpp | 54 +++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 5b9e984df96..8139c897a01 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -60,25 +60,28 @@ public: { if (arguments.size() != 2 && arguments.size() != 3) throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be 2 or 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}", + getName(), + toString(arguments.size())); if (!WhichDataType(arguments[0].type).isString()) throw Exception( - "Illegal type " + arguments[0].type->getName() + " of 1 argument of function " + getName() - + ". Must be string", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 1 argument of function {}. Must be string", + arguments[0].type->getName(), + getName()); if (!WhichDataType(arguments[1].type).isDateOrDateTime()) throw Exception( - "Illegal type " + arguments[1].type->getName() + " of 2 argument of function " + getName() - + "Must be a date or a date with time", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 2 argument of function {}. Must be a date or a date with time", + arguments[1].type->getName(), + getName()); if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isString()) throw Exception( - "Illegal type " + arguments[2].type->getName() + " of 3 argument of function " + getName() - + "Must be string", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of 3 argument of function {}. Must be string", + arguments[2].type->getName(), + getName()); return std::make_shared(); } @@ -91,9 +94,10 @@ public: || (res = executeType(arguments, result_type)) || (res = executeType(arguments, result_type)))) throw Exception( - "Illegal column " + arguments[1].column->getName() + " of function " + getName() - + ", must be Date or DateTime.", - ErrorCodes::ILLEGAL_COLUMN); + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of function {], must be Date or DateTime.", + arguments[1].column->getName(), + getName()); return res; } @@ -107,20 +111,22 @@ public: const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); if (!datepart_column) - throw Exception("Illegal column " + arguments[0].column->getName() - + " of first ('datepart') argument of function " + getName() - + ". Must be constant string.", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first ('datepart') argument of function {}. Must be constant string.", + arguments[0].column->getName(), + getName()); using T = typename ActionValueTypeMap::ActionValueType; auto datepart_writer = DatePartWriter(); String datepart = datepart_column->getValue(); if (!datepart_writer.isCorrectDatePart(datepart)) - throw Exception("Illegal value " + datepart - + " of first ('format') argument of function " + getName() - + ". Check documentation.", - ErrorCodes::BAD_ARGUMENTS); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Illegal value {} of first ('format') argument of function {}. Check documentation", + datepart, + getName()); const DateLUTImpl * time_zone_tmp; if (std::is_same_v || std::is_same_v) From a041cc9db0aa2c705a2a11146b006dba08b98c86 Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:32:37 +0300 Subject: [PATCH 023/491] Fix problem with offsets --- src/Functions/dateName.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 8139c897a01..28f9361fd97 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -166,6 +166,7 @@ public: datepart_writer.writeDatePart(pos, datepart, vec[i], time_zone); } dst_offsets[i] = pos - begin; + ++pos; } dst_data.resize(pos - begin); return col_res; @@ -275,25 +276,25 @@ private: { *target = value + '0'; target += 2; - *target++ = 0; + *target = 0; } else if (value < 100) { writeNumber2(target, value); target += 3; - *target++ = 0; + *target = 0; } else if (value < 1000) { writeNumber3(target, value); target += 4; - *target++ = 0; + *target = 0; } else if (value < 10000) { writeNumber4(target, value); target += 5; - *target++ = 0; + *target = 0; } else { From 58ed68c28ef6eaa6ccba0bafa4d9dde304fcdaa4 Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:44:56 +0300 Subject: [PATCH 024/491] Fix writeNumber2 --- src/Functions/dateName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 28f9361fd97..3d0211a6bbf 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -313,7 +313,7 @@ private: static inline void writeNumber3(char * p, T v) { writeNumber2(p, v / 10); - p[2] += v % 10; + p[2] = v % 10 + '0'; } template From 4c37eba92b402d5bb522e595479be2836aec9686 Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:46:46 +0300 Subject: [PATCH 025/491] Improve code readability --- src/Functions/dateName.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 3d0211a6bbf..da45fcb7dcb 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -276,25 +276,25 @@ private: { *target = value + '0'; target += 2; - *target = 0; + *target = '\0'; } else if (value < 100) { writeNumber2(target, value); target += 3; - *target = 0; + *target = '\0'; } else if (value < 1000) { writeNumber3(target, value); target += 4; - *target = 0; + *target = '\0'; } else if (value < 10000) { writeNumber4(target, value); target += 5; - *target = 0; + *target = '\0'; } else { From be97fd358e36d15baef40cffe0ae6f1abde4432a Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:55:46 +0300 Subject: [PATCH 026/491] Fix problem with 3rd argument being const --- src/Functions/dateName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index da45fcb7dcb..1d6f5352a78 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -51,7 +51,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 2}; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } From 107357f96194919490a25c017ea900b5e15bcbd9 Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 20:59:10 +0300 Subject: [PATCH 027/491] Fix tests data --- tests/queries/0_stateless/01811_datename.sql | 23 +++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/01811_datename.sql b/tests/queries/0_stateless/01811_datename.sql index 87ece62f78a..042bf0a121a 100644 --- a/tests/queries/0_stateless/01811_datename.sql +++ b/tests/queries/0_stateless/01811_datename.sql @@ -10,16 +10,16 @@ SELECT dateName('minute', toDateTime('2021-04-14 11:22:33')); SELECT dateName('second', toDateTime('2021-04-14 11:22:33')); -SELECT dateName('year', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('quarter', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('month', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('dayofyear', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('day', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('week', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('weekday', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('hour', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('minute', toDateTime64('2021-04-14 11:22:33')); -SELECT dateName('second', toDateTime64('2021-04-14 11:22:33')); +SELECT dateName('year', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('quarter', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('month', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('dayofyear', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('day', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('week', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('weekday', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('hour', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('minute', toDateTime64('2021-04-14 11:22:33', 3)); +SELECT dateName('second', toDateTime64('2021-04-14 11:22:33', 3)); SELECT dateName('year', toDate('2021-04-14')); @@ -29,9 +29,6 @@ SELECT dateName('dayofyear', toDate('2021-04-14')); SELECT dateName('day', toDate('2021-04-14')); SELECT dateName('week', toDate('2021-04-14')); SELECT dateName('weekday', toDate('2021-04-14')); -SELECT dateName('hour', toDate('2021-04-14')); -SELECT dateName('minute', toDate('2021-04-14')); -SELECT dateName('second', toDate('2021-04-14')); SELECT dateName('day', toDateTime('2021-04-14 11:22:33'), 'Europe/Moscow'), From 7e741338f2f69a472e49c81694f9ccc469e19e66 Mon Sep 17 00:00:00 2001 From: dankondr Date: Fri, 16 Apr 2021 21:24:38 +0300 Subject: [PATCH 028/491] Fix typo --- src/Functions/dateName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 1d6f5352a78..b493dc7f7ab 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -298,7 +298,7 @@ private: } else { - throw Exception("Illegal value of second ('datetime') argument of function datePart. Check documentation.", + throw Exception("Illegal value of second ('datetime') argument of function dateName. Check documentation.", ErrorCodes::BAD_ARGUMENTS); } } From ffcfe5f9a1e9ba2eec6e8995186c5e1ebaea4414 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 26 Apr 2021 13:34:44 +0000 Subject: [PATCH 029/491] Progress bar for file table engine --- src/IO/Progress.h | 33 +++++++ src/Interpreters/Context.h | 12 ++- src/Processors/Sources/SourceWithProgress.h | 8 +- src/Storages/StorageFile.cpp | 96 ++++++++++++++++++- src/Storages/StorageFile.h | 8 +- src/TableFunctions/ITableFunctionFileLike.cpp | 3 +- 6 files changed, 148 insertions(+), 12 deletions(-) diff --git a/src/IO/Progress.h b/src/IO/Progress.h index 64bf3a404af..2f69396dd5d 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -5,6 +5,8 @@ #include #include +#include + namespace DB { @@ -45,6 +47,37 @@ struct WriteProgress : written_rows(written_rows_), written_bytes(written_bytes_) {} }; +/// Track progress of processing one or multiple files via File table engine. +struct FileTableEngineProgress +{ + /// Track elapsed time. + Stopwatch watch; + size_t total_bytes_to_process; + + /// FileTableEngineProgress struct can be accessed from Context via const reference. + /// These fields are allowed to be updated in a progress callback. + mutable std::atomic processed_bytes; + mutable std::atomic processed_rows; + + FileTableEngineProgress() : total_bytes_to_process(0), processed_bytes(0) {} + + FileTableEngineProgress(const FileTableEngineProgress & other) + : watch(other.watch) + , total_bytes_to_process(other.total_bytes_to_process) + , processed_bytes(other.processed_bytes.load()) + , processed_rows(other.processed_rows.load()) {} + + FileTableEngineProgress & operator=(FileTableEngineProgress other) + { + watch = other.watch; + total_bytes_to_process = other.total_bytes_to_process; + processed_bytes = other.processed_bytes.load(); + processed_rows = other.processed_rows.load(); + return *this; + } +}; + + /** Progress of query execution. * Values, transferred over network are deltas - how much was done after previously sent value. * The same struct is also used for summarized values. diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 680ee7c779f..adcb7cfe5eb 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -181,10 +181,13 @@ private: std::shared_ptr access; std::shared_ptr initial_row_policy; String current_database; - Settings settings; /// Setting for query execution. + Settings settings; /// Setting for query execution. + using ProgressCallback = std::function; - ProgressCallback progress_callback; /// Callback for tracking progress of query execution. - QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. + ProgressCallback progress_callback; /// Callback for tracking progress of query execution. + FileTableEngineProgress file_progress; /// Progress data to track processing of one or multiple files for File table engine. + + QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. StorageID insertion_table = StorageID::createEmpty(); /// Saved insertion table in query context String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. @@ -581,6 +584,9 @@ public: /// Used in InterpreterSelectQuery to pass it to the IBlockInputStream. ProgressCallback getProgressCallback() const; + const FileTableEngineProgress & getFileTableEngineProgress() { return file_progress; } + void setFileTableEngineApproxBytesToProcess(size_t num_bytes) { file_progress.total_bytes_to_process = num_bytes; } + /** Set in executeQuery and InterpreterSelectQuery. Then it is used in IBlockInputStream, * to update and monitor information about the total number of resources spent for the query. */ diff --git a/src/Processors/Sources/SourceWithProgress.h b/src/Processors/Sources/SourceWithProgress.h index 3aa7a81f418..3b283009c73 100644 --- a/src/Processors/Sources/SourceWithProgress.h +++ b/src/Processors/Sources/SourceWithProgress.h @@ -52,20 +52,24 @@ public: void setLeafLimits(const SizeLimits & leaf_limits_) final {leaf_limits = leaf_limits_; } void setQuota(const std::shared_ptr & quota_) final { quota = quota_; } void setProcessListElement(QueryStatus * elem) final { process_list_elem = elem; } - void setProgressCallback(const ProgressCallback & callback) final { progress_callback = callback; } void addTotalRowsApprox(size_t value) final { total_rows_approx += value; } + /// This method might be overriden, if, during query execution, there is a Source, that needs + /// to add one more progress callback. + void setProgressCallback(const ProgressCallback & callback) override { progress_callback = callback; } + protected: /// Call this method to provide information about progress. void progress(const Progress & value); void work() override; + ProgressCallback progress_callback; + private: StreamLocalLimits limits; SizeLimits leaf_limits; std::shared_ptr quota; - ProgressCallback progress_callback; QueryStatus * process_list_elem = nullptr; /// The approximate total number of rows to read. For progress bar. diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 14b91d29805..81679a2916d 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -38,6 +38,9 @@ #include #include #include +#include +#include + namespace fs = std::filesystem; @@ -65,7 +68,7 @@ namespace /* Recursive directory listing with matched paths as a result. * Have the same method in StorageHDFS. */ -std::vector listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match) +std::vector listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match, size_t & total_bytes_to_read) { const size_t first_glob = for_match.find_first_of("*?{"); @@ -94,6 +97,7 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo { if (re2::RE2::FullMatch(file_name, matcher)) { + total_bytes_to_read += fs::file_size(it->path()); result.push_back(it->path().string()); } } @@ -102,7 +106,7 @@ std::vector listFilesWithRegexpMatching(const std::string & path_fo if (re2::RE2::FullMatch(file_name, matcher)) { /// Recursion depth is limited by pattern. '*' works only for depth = 1, for depth = 2 pattern path is '*/*'. So we do not need additional check. - Strings result_part = listFilesWithRegexpMatching(full_path + "/", suffix_with_globs.substr(next_slash)); + Strings result_part = listFilesWithRegexpMatching(full_path + "/", suffix_with_globs.substr(next_slash), total_bytes_to_read); std::move(result_part.begin(), result_part.end(), std::back_inserter(result)); } } @@ -131,7 +135,7 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di } } -Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, ContextPtr context) +Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, ContextPtr context, size_t & total_bytes_to_read) { String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); Poco::Path poco_path = Poco::Path(table_path); @@ -141,9 +145,12 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user Strings paths; const String path = poco_path.absolute().toString(); if (path.find_first_of("*?{") == std::string::npos) + { + total_bytes_to_read += fs::file_size(path); paths.push_back(path); + } else - paths = listFilesWithRegexpMatching("/", path); + paths = listFilesWithRegexpMatching("/", path, total_bytes_to_read); for (const auto & cur_path : paths) checkCreationIsAllowed(context, user_files_absolute_path, cur_path); @@ -177,7 +184,7 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us : StorageFile(args) { is_db_table = false; - paths = getPathsList(table_path_, user_files_path, args.getContext()); + paths = getPathsList(table_path_, user_files_path, args.getContext(), total_bytes_to_read); if (args.format_name == "Distributed") { @@ -329,6 +336,7 @@ public: Chunk generate() override { + //setFileProgressCallback(); while (!finished_generate) { /// Open file lazily on first read. This is needed to avoid too many open files from different streams. @@ -421,6 +429,80 @@ public: return {}; } + + void setProgressCallback(const ProgressCallback & callback) override + { + /// Add file progress callback only for clickhouse-local. + if (context->getApplicationType() != Context::ApplicationType::LOCAL) + { + progress_callback = callback; + return; + } + + auto file_progress_callback = [this](const Progress & progress) + { + static size_t increment = 0; + static const char * indicators[8] = + { + "\033[1;30m→\033[0m", + "\033[1;31m↘\033[0m", + "\033[1;32m↓\033[0m", + "\033[1;33m↙\033[0m", + "\033[1;34m←\033[0m", + "\033[1;35m↖\033[0m", + "\033[1;36m↑\033[0m", + "\033[1m↗\033[0m", + }; + size_t terminal_width = getTerminalWidth(); + + auto & file_progress = context->getFileTableEngineProgress(); + WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); + + /// Output progress bar one line lower. + if (!file_progress.processed_bytes) + message << std::string(terminal_width, ' '); + + file_progress.processed_bytes += progress.read_bytes; + file_progress.processed_rows += progress.read_rows; + + /// Display progress bar only if .25 seconds have passed since query execution start. + size_t elapsed_ns = file_progress.watch.elapsed(); + if (elapsed_ns > 25000000 && progress.read_bytes > 0) + { + message << '\r'; + const char * indicator = indicators[increment % 8]; + size_t prefix_size = message.count(); + + message << indicator << " Progress: "; + message << formatReadableQuantity(file_progress.processed_rows) << " rows, "; + message << formatReadableSizeWithDecimalSuffix(file_progress.processed_bytes) << " bytes. "; + + size_t written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 1); /// Don't count invisible output (escape sequences). + ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); + std::string bar = UnicodeBar::render(UnicodeBar::getWidth(file_progress.processed_bytes, 0, file_progress.total_bytes_to_process, width_of_progress_bar)); + + message << "\033[0;32m" << bar << "\033[0m"; + + if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) + message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); + + message << ' ' << std::min((100 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; + } + ++increment; + }; + + /// Progress callback can be added via context or via method in SourceWithProgress. + /// In executeQuery a callback from context is wrapped into another + /// progress callback and then passed to SourceWithProgress. Here another callback is + /// added to avoid overriding previous callbacks or avoid other callbacks overriding this one. + progress_callback = [callback, file_progress_callback](const Progress & progress) + { + callback(progress); + file_progress_callback(progress); + }; + } + + private: std::shared_ptr storage; StorageMetadataPtr metadata_snapshot; @@ -483,6 +565,10 @@ Pipe StorageFile::read( Pipes pipes; pipes.reserve(num_streams); + /// For clickhouse-local add progress callback to display in a progress bar. + if (context->getApplicationType() == Context::ApplicationType::LOCAL) + context->setFileTableEngineApproxBytesToProcess(total_bytes_to_read); + for (size_t i = 0; i < num_streams; ++i) { const auto get_columns_for_format = [&]() -> ColumnsDescription diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index a277dda7cc0..fc69f29cd20 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -61,7 +61,7 @@ public: NamesAndTypesList getVirtuals() const override; - static Strings getPathsList(const String & table_path, const String & user_files_path, ContextPtr context); + static Strings getPathsList(const String & table_path, const String & user_files_path, ContextPtr context, size_t & total_bytes_to_read); /// Check if the format is column-oriented. /// Is is useful because column oriented formats could effectively skip unknown columns @@ -85,6 +85,9 @@ protected: private: explicit StorageFile(CommonArguments args); + /// For clickhouse-local query display progress of processed files. + static void addProgressCallback(ContextPtr context); + std::string format_name; // We use format settings from global context + CREATE query for File table // function -- in this case, format_settings is set. @@ -106,6 +109,9 @@ private: mutable std::shared_timed_mutex rwlock; Poco::Logger * log = &Poco::Logger::get("StorageFile"); + + /// Approximate number of bytes to read. Needed for progress bar. + size_t total_bytes_to_read = 0; }; } diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 44a917a0f00..f3d6905d1a7 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -77,7 +77,8 @@ ColumnsDescription ITableFunctionFileLike::getActualTableStructure(ContextPtr co if (structure.empty()) { assert(getName() == "file" && format == "Distributed"); - Strings paths = StorageFile::getPathsList(filename, context->getUserFilesPath(), context); + size_t total_bytes_to_read = 0; + Strings paths = StorageFile::getPathsList(filename, context->getUserFilesPath(), context, total_bytes_to_read); if (paths.empty()) throw Exception("Cannot get table structure from file, because no files match specified name", ErrorCodes::INCORRECT_FILE_NAME); auto read_stream = StorageDistributedDirectoryMonitor::createStreamFromFile(paths[0]); From 5efd075ad2ba8d61f8326ca013ace90e46951bed Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 26 Apr 2021 22:38:56 +0000 Subject: [PATCH 030/491] Slightly bette --- src/Storages/StorageFile.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 81679a2916d..86eb3dbd190 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -336,7 +336,6 @@ public: Chunk generate() override { - //setFileProgressCallback(); while (!finished_generate) { /// Open file lazily on first read. This is needed to avoid too many open files from different streams. @@ -458,7 +457,6 @@ public: auto & file_progress = context->getFileTableEngineProgress(); WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); - /// Output progress bar one line lower. if (!file_progress.processed_bytes) message << std::string(terminal_width, ' '); @@ -467,11 +465,12 @@ public: /// Display progress bar only if .25 seconds have passed since query execution start. size_t elapsed_ns = file_progress.watch.elapsed(); - if (elapsed_ns > 25000000 && progress.read_bytes > 0) + if (elapsed_ns > 25000000) { message << '\r'; const char * indicator = indicators[increment % 8]; size_t prefix_size = message.count(); + size_t processed_bytes = file_progress.processed_bytes.load(); message << indicator << " Progress: "; message << formatReadableQuantity(file_progress.processed_rows) << " rows, "; @@ -479,14 +478,18 @@ public: size_t written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 1); /// Don't count invisible output (escape sequences). ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); - std::string bar = UnicodeBar::render(UnicodeBar::getWidth(file_progress.processed_bytes, 0, file_progress.total_bytes_to_process, width_of_progress_bar)); + /// total_bytes_to_read is approximate, since its amount is taken as file size (or sum of all file sizes + /// from paths, generated for file table engine). And progress.read_bytes is counted accorging to columns types. + size_t total_bytes_corrected = std::max(processed_bytes, file_progress.total_bytes_to_process); + + std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); message << "\033[0;32m" << bar << "\033[0m"; if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); - message << ' ' << std::min((100 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; + message << ' ' << std::min((99 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; } ++increment; }; From 1135c47082d6ea47a1e946f7d01600e2b083184d Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Apr 2021 09:54:12 +0000 Subject: [PATCH 031/491] Fixes --- src/Storages/StorageFile.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 86eb3dbd190..65b7bc14507 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -146,7 +146,9 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user const String path = poco_path.absolute().toString(); if (path.find_first_of("*?{") == std::string::npos) { - total_bytes_to_read += fs::file_size(path); + std::error_code error; + if (fs::exists(path)) + total_bytes_to_read += fs::file_size(path, error); paths.push_back(path); } else @@ -465,7 +467,7 @@ public: /// Display progress bar only if .25 seconds have passed since query execution start. size_t elapsed_ns = file_progress.watch.elapsed(); - if (elapsed_ns > 25000000) + if (elapsed_ns > 25000000 && progress.read_bytes > 0) { message << '\r'; const char * indicator = indicators[increment % 8]; @@ -483,13 +485,16 @@ public: /// from paths, generated for file table engine). And progress.read_bytes is counted accorging to columns types. size_t total_bytes_corrected = std::max(processed_bytes, file_progress.total_bytes_to_process); - std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); - message << "\033[0;32m" << bar << "\033[0m"; + if (width_of_progress_bar > 0) + { + std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); + message << "\033[0;32m" << bar << "\033[0m"; - if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) - message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); + if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) + message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); - message << ' ' << std::min((99 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; + message << ' ' << std::min((99 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; + } } ++increment; }; From 3a0372535ea2f0b466e832163b479889bbcd6352 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Apr 2021 13:19:52 +0000 Subject: [PATCH 032/491] Show progress only with --progress option --- programs/local/LocalServer.cpp | 5 +++++ src/Interpreters/Context.h | 4 ++++ src/Storages/StorageFile.cpp | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f680c2c2da6..f4f973f2db1 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -388,6 +388,8 @@ void LocalServer::processQueries() CurrentThread::QueryScope query_scope_holder(context); bool echo_queries = config().hasOption("echo") || config().hasOption("verbose"); + if (config().hasOption("progress")) + context->setRenderProgress(); std::exception_ptr exception; for (const auto & query : queries) @@ -540,6 +542,7 @@ void LocalServer::init(int argc, char ** argv) ("output-format", po::value(), "default output format") ("stacktrace", "print stack traces of exceptions") + ("progress", "show progress for File table engine") ("echo", "print query before execution") ("verbose", "print query and other debugging info") ("logger.console", po::value()->implicit_value(true), "Log to console") @@ -597,6 +600,8 @@ void LocalServer::init(int argc, char ** argv) if (options.count("stacktrace")) config().setBool("stacktrace", true); + if (options.count("progress")) + config().setBool("progress", true); if (options.count("echo")) config().setBool("echo", true); if (options.count("verbose")) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index adcb7cfe5eb..6771c59f763 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -186,6 +186,7 @@ private: using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. FileTableEngineProgress file_progress; /// Progress data to track processing of one or multiple files for File table engine. + bool render_progress = false; QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. StorageID insertion_table = StorageID::createEmpty(); /// Saved insertion table in query context @@ -584,6 +585,9 @@ public: /// Used in InterpreterSelectQuery to pass it to the IBlockInputStream. ProgressCallback getProgressCallback() const; + void setRenderProgress() { render_progress = true; } + bool needRenderProgress() const { return render_progress; } + const FileTableEngineProgress & getFileTableEngineProgress() { return file_progress; } void setFileTableEngineApproxBytesToProcess(size_t num_bytes) { file_progress.total_bytes_to_process = num_bytes; } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 65b7bc14507..66593e1b37a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -434,7 +434,7 @@ public: void setProgressCallback(const ProgressCallback & callback) override { /// Add file progress callback only for clickhouse-local. - if (context->getApplicationType() != Context::ApplicationType::LOCAL) + if (!context->needRenderProgress() || context->getApplicationType() != Context::ApplicationType::LOCAL) { progress_callback = callback; return; From 6675ed0681c0d077143e1dd473dd1d451446ef85 Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 17:26:00 +0300 Subject: [PATCH 033/491] Add tests --- .../0_stateless/01811_datename.reference | 47 +++++++++++++++++++ tests/queries/0_stateless/01811_datename.sql | 4 +- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01811_datename.reference diff --git a/tests/queries/0_stateless/01811_datename.reference b/tests/queries/0_stateless/01811_datename.reference new file mode 100644 index 00000000000..8bfdac794ba --- /dev/null +++ b/tests/queries/0_stateless/01811_datename.reference @@ -0,0 +1,47 @@ +2021 +2 +April +104 +14 +15 +Wednesday +11 +22 +33 +2021 +2 +April +104 +14 +15 +Wednesday +11 +22 +33 +2021 +2 +April +104 +14 +15 +Wednesday +14 11 +Monday +Tuesday +Wednesday +Thursday +Friday +Saturday +Sunday +January +February +March +April +May +June +July +August +September +October +November +December \ No newline at end of file diff --git a/tests/queries/0_stateless/01811_datename.sql b/tests/queries/0_stateless/01811_datename.sql index 042bf0a121a..7ecc7699462 100644 --- a/tests/queries/0_stateless/01811_datename.sql +++ b/tests/queries/0_stateless/01811_datename.sql @@ -31,8 +31,8 @@ SELECT dateName('week', toDate('2021-04-14')); SELECT dateName('weekday', toDate('2021-04-14')); -SELECT dateName('day', toDateTime('2021-04-14 11:22:33'), 'Europe/Moscow'), - dateName('day', toDateTime('2021-04-14 11:22:33'), 'UTC'); +SELECT dateName('hour', toDateTime('2021-04-14 11:22:33'), 'Europe/Moscow'), + dateName('hour', toDateTime('2021-04-14 11:22:33'), 'UTC'); SELECT dateName('day', toDate('2021-04-12')); SELECT dateName('day', toDate('2021-04-13')); From 1365a2c82e74a6782f3de8c16525a08c0682a9c2 Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 17:26:11 +0300 Subject: [PATCH 034/491] Fix include --- src/Functions/dateName.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index b493dc7f7ab..e08710f35dc 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include From b8db093a656a423dbac4d4547fa0f29d7c95a345 Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 17:48:16 +0300 Subject: [PATCH 035/491] Fix tests --- tests/queries/0_stateless/01811_datename.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01811_datename.sql b/tests/queries/0_stateless/01811_datename.sql index 7ecc7699462..88afc656ec1 100644 --- a/tests/queries/0_stateless/01811_datename.sql +++ b/tests/queries/0_stateless/01811_datename.sql @@ -34,13 +34,13 @@ SELECT dateName('weekday', toDate('2021-04-14')); SELECT dateName('hour', toDateTime('2021-04-14 11:22:33'), 'Europe/Moscow'), dateName('hour', toDateTime('2021-04-14 11:22:33'), 'UTC'); -SELECT dateName('day', toDate('2021-04-12')); -SELECT dateName('day', toDate('2021-04-13')); -SELECT dateName('day', toDate('2021-04-14')); -SELECT dateName('day', toDate('2021-04-15')); -SELECT dateName('day', toDate('2021-04-16')); -SELECT dateName('day', toDate('2021-04-17')); -SELECT dateName('day', toDate('2021-04-18')); +SELECT dateName('weekday', toDate('2021-04-12')); +SELECT dateName('weekday', toDate('2021-04-13')); +SELECT dateName('weekday', toDate('2021-04-14')); +SELECT dateName('weekday', toDate('2021-04-15')); +SELECT dateName('weekday', toDate('2021-04-16')); +SELECT dateName('weekday', toDate('2021-04-17')); +SELECT dateName('weekday', toDate('2021-04-18')); SELECT dateName('month', toDate('2021-01-14')); SELECT dateName('month', toDate('2021-02-14')); From e11b0b18a554742a3d66a8c0a18f03d04d68276f Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 18:29:23 +0300 Subject: [PATCH 036/491] Fix tests --- tests/queries/0_stateless/01811_datename.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01811_datename.reference b/tests/queries/0_stateless/01811_datename.reference index 8bfdac794ba..df62af8ffed 100644 --- a/tests/queries/0_stateless/01811_datename.reference +++ b/tests/queries/0_stateless/01811_datename.reference @@ -25,7 +25,7 @@ April 14 15 Wednesday -14 11 +11 8 Monday Tuesday Wednesday @@ -44,4 +44,4 @@ August September October November -December \ No newline at end of file +December From 637ddc575f745f7df50247f27d3c148545711f46 Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 19:06:07 +0300 Subject: [PATCH 037/491] Fix tests --- tests/queries/0_stateless/01811_datename.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01811_datename.reference b/tests/queries/0_stateless/01811_datename.reference index df62af8ffed..1b4d4b785cf 100644 --- a/tests/queries/0_stateless/01811_datename.reference +++ b/tests/queries/0_stateless/01811_datename.reference @@ -25,7 +25,7 @@ April 14 15 Wednesday -11 8 +11 8 Monday Tuesday Wednesday From 780f58abc40dc21782174c2dff03123eedc078be Mon Sep 17 00:00:00 2001 From: dankondr Date: Tue, 27 Apr 2021 19:54:05 +0300 Subject: [PATCH 038/491] Fix code style --- src/Functions/dateName.cpp | 51 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index e08710f35dc..2d9ecf0c124 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -1,8 +1,8 @@ -#include +#include #include #include #include -#include +#include #include #include @@ -14,8 +14,8 @@ #include -#include #include +#include #include @@ -43,10 +43,7 @@ public: static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } bool useDefaultImplementationForConstants() const override { return true; } @@ -85,12 +82,13 @@ public: } ColumnPtr executeImpl( - const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, [[maybe_unused]] size_t input_rows_count) const override + const ColumnsWithTypeAndName & arguments, + const DataTypePtr & result_type, + [[maybe_unused]] size_t input_rows_count) const override { ColumnPtr res; - if (!((res = executeType(arguments, result_type)) - || (res = executeType(arguments, result_type)) + if (!((res = executeType(arguments, result_type)) || (res = executeType(arguments, result_type)) || (res = executeType(arguments, result_type)))) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -181,10 +179,7 @@ private: datepart_functions.at(datepart)(target, source, timezone); } - bool isCorrectDatePart(const String &datepart) - { - return datepart_functions.find(datepart) != datepart_functions.end(); - } + bool isCorrectDatePart(const String & datepart) { return datepart_functions.find(datepart) != datepart_functions.end(); } private: const std::unordered_map datepart_functions = { @@ -213,12 +208,19 @@ private: static inline void writeMonth(char *& target, Time source, const DateLUTImpl & timezone) { const auto month = ToMonthImpl::execute(source, timezone); - const String monthnames[12] = { - "January", "February", "March", - "April", "May", "June", - "July", "August", "September", - "October", "November", "December" - }; + const String monthnames[12] + = {"January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December"}; writeString(target, monthnames[month - 1]); } @@ -240,9 +242,7 @@ private: static inline void writeWeekday(char *& target, Time source, const DateLUTImpl & timezone) { const auto day = ToDayOfWeekImpl::execute(source, timezone); - const String daynames[12] = { - "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday" - }; + const String daynames[12] = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"}; writeString(target, daynames[day - 1]); } @@ -297,8 +297,9 @@ private: } else { - throw Exception("Illegal value of second ('datetime') argument of function dateName. Check documentation.", - ErrorCodes::BAD_ARGUMENTS); + throw Exception( + "Illegal value of second ('datetime') argument of function dateName. Check documentation.", + ErrorCodes::BAD_ARGUMENTS); } } From 563f8a97c6d4c453b7e8cd1df7adf8fdf9e2a1eb Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Apr 2021 17:01:41 +0000 Subject: [PATCH 039/491] Fix build, typos check --- src/IO/Progress.h | 2 +- src/Processors/Sources/SourceWithProgress.h | 2 +- src/Storages/StorageFile.cpp | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/IO/Progress.h b/src/IO/Progress.h index 2f69396dd5d..a2204939267 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -59,7 +59,7 @@ struct FileTableEngineProgress mutable std::atomic processed_bytes; mutable std::atomic processed_rows; - FileTableEngineProgress() : total_bytes_to_process(0), processed_bytes(0) {} + FileTableEngineProgress() : total_bytes_to_process(0), processed_bytes(0), processed_rows(0) {} FileTableEngineProgress(const FileTableEngineProgress & other) : watch(other.watch) diff --git a/src/Processors/Sources/SourceWithProgress.h b/src/Processors/Sources/SourceWithProgress.h index 3b283009c73..37456eb970e 100644 --- a/src/Processors/Sources/SourceWithProgress.h +++ b/src/Processors/Sources/SourceWithProgress.h @@ -54,7 +54,7 @@ public: void setProcessListElement(QueryStatus * elem) final { process_list_elem = elem; } void addTotalRowsApprox(size_t value) final { total_rows_approx += value; } - /// This method might be overriden, if, during query execution, there is a Source, that needs + /// This method might be overridden, if, during query execution, there is a Source, that needs /// to add one more progress callback. void setProgressCallback(const ProgressCallback & callback) override { progress_callback = callback; } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 66593e1b37a..91fb3078b91 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -456,7 +457,7 @@ public: }; size_t terminal_width = getTerminalWidth(); - auto & file_progress = context->getFileTableEngineProgress(); + const auto & file_progress = context->getFileTableEngineProgress(); WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); if (!file_progress.processed_bytes) From 675d3e6c880dd009c32412f0bc5008ce95d5e3b4 Mon Sep 17 00:00:00 2001 From: dankondr Date: Wed, 28 Apr 2021 20:09:46 +0300 Subject: [PATCH 040/491] Fix code style --- src/Functions/dateName.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 2d9ecf0c124..d1551dd2442 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -29,7 +29,8 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -namespace { +namespace +{ template struct ActionValueTypeMap {}; template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; From c33f358a9acf68746f8f19411576869a5c9bc21b Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 28 Apr 2021 16:25:14 +0000 Subject: [PATCH 041/491] Better way to track progress, display progress for ch-client --- programs/client/Client.cpp | 3 + src/IO/Progress.h | 18 ++-- src/IO/ReadBufferFromFileDescriptor.cpp | 89 +++++++++++++++++++ src/IO/ReadBufferFromFileDescriptor.h | 4 + src/Interpreters/Context.h | 6 +- src/Processors/Sources/SourceWithProgress.h | 8 +- src/Storages/StorageFile.cpp | 94 ++------------------- src/Storages/StorageFile.h | 5 +- 8 files changed, 119 insertions(+), 108 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 1aec3677b41..7756d40fe47 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1856,6 +1856,9 @@ private: /// Send data read from stdin. try { + if (need_render_progress) + std_in.setProgressCallback(context); + sendDataFrom(std_in, sample, columns_description); } catch (Exception & e) diff --git a/src/IO/Progress.h b/src/IO/Progress.h index a2204939267..240564849e3 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -47,32 +47,30 @@ struct WriteProgress : written_rows(written_rows_), written_bytes(written_bytes_) {} }; -/// Track progress of processing one or multiple files via File table engine. -struct FileTableEngineProgress +/// Track progress of processing file base buffer. +/// Used to display progress of loading data from stdin, for file table engine, etc. +struct FileProgress { /// Track elapsed time. Stopwatch watch; size_t total_bytes_to_process; - /// FileTableEngineProgress struct can be accessed from Context via const reference. + /// FileProgress lies in Context and accessed via const reference. /// These fields are allowed to be updated in a progress callback. mutable std::atomic processed_bytes; - mutable std::atomic processed_rows; - FileTableEngineProgress() : total_bytes_to_process(0), processed_bytes(0), processed_rows(0) {} + FileProgress() : total_bytes_to_process(0), processed_bytes(0) {} - FileTableEngineProgress(const FileTableEngineProgress & other) + FileProgress(const FileProgress & other) : watch(other.watch) , total_bytes_to_process(other.total_bytes_to_process) - , processed_bytes(other.processed_bytes.load()) - , processed_rows(other.processed_rows.load()) {} + , processed_bytes(other.processed_bytes.load()) {} - FileTableEngineProgress & operator=(FileTableEngineProgress other) + FileProgress & operator=(FileProgress other) { watch = other.watch; total_bytes_to_process = other.total_bytes_to_process; processed_bytes = other.processed_bytes.load(); - processed_rows = other.processed_rows.load(); return *this; } }; diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index dd5d9e67cd7..786d944c0ae 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -6,7 +6,14 @@ #include #include #include +#include #include +#include +#include +#include +#include + +#define CLEAR_TO_END_OF_LINE "\033[K" namespace ProfileEvents @@ -32,6 +39,7 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; extern const int CANNOT_SEEK_THROUGH_FILE; extern const int CANNOT_SELECT; + extern const int CANNOT_FSTAT; } @@ -170,4 +178,85 @@ bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) return res > 0; } + +off_t ReadBufferFromFileDescriptor::size() +{ + struct stat buf; + int res = fstat(fd, &buf); + if (-1 == res) + throwFromErrnoWithPath("Cannot execute fstat " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSTAT); + return buf.st_size; +} + + +void ReadBufferFromFileDescriptor::setProgressCallback(ContextPtr context) +{ + /// Keep file progress and total bytes to process in context and not in readBuffer, because + /// multiple files might share the same progress (for example, for file table engine when globs are used) + /// and total_bytes_to_process will contain sum of sizes of all files. + + if (!context->getFileProgress().total_bytes_to_process) + context->setFileTotalBytesToProcess(size()); + + setProfileCallback([context](const ProfileInfo & progress) + { + static size_t increment = 0; + static const char * indicators[8] = + { + "\033[1;30m→\033[0m", + "\033[1;31m↘\033[0m", + "\033[1;32m↓\033[0m", + "\033[1;33m↙\033[0m", + "\033[1;34m←\033[0m", + "\033[1;35m↖\033[0m", + "\033[1;36m↑\033[0m", + "\033[1m↗\033[0m", + }; + size_t terminal_width = getTerminalWidth(); + WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); + + const auto & file_progress = context->getFileProgress(); + + if (!file_progress.processed_bytes) + message << std::string(terminal_width, ' '); + message << '\r'; + file_progress.processed_bytes += progress.bytes_read; + + const char * indicator = indicators[increment % 8]; + size_t prefix_size = message.count(); + size_t processed_bytes = file_progress.processed_bytes.load(); + + message << indicator << " Progress: "; + message << formatReadableSizeWithDecimalSuffix(file_progress.processed_bytes); + message << " from " << formatReadableSizeWithDecimalSuffix(file_progress.total_bytes_to_process) << " bytes. "; + + /// Display progress bar only if .25 seconds have passed since query execution start. + size_t elapsed_ns = file_progress.watch.elapsed(); + if (elapsed_ns > 25000000) + { + size_t written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 1); /// Don't count invisible output (escape sequences). + ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); + + size_t total_bytes_corrected = std::max(processed_bytes, file_progress.total_bytes_to_process); + + if (width_of_progress_bar > 0 && progress.bytes_read > 0) + { + std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); + message << "\033[0;32m" << bar << "\033[0m"; + + if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) + message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); + } + + /// Underestimate percentage a bit to avoid displaying 100%. + message << ' ' << (99 * file_progress.processed_bytes / file_progress.total_bytes_to_process) << '%'; + } + + message << CLEAR_TO_END_OF_LINE; + message.next(); + + ++increment; + }); +} + } diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 0779b215067..bf22bb3d4a3 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -38,6 +39,9 @@ public: /// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen. off_t seek(off_t off, int whence) override; + off_t size(); + + void setProgressCallback(ContextPtr context); private: /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. bool poll(size_t timeout_microseconds); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6771c59f763..bc23977c5ec 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -185,7 +185,7 @@ private: using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. - FileTableEngineProgress file_progress; /// Progress data to track processing of one or multiple files for File table engine. + FileProgress file_progress; /// Progress data to track processing of file based buffer(s). bool render_progress = false; QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. @@ -588,8 +588,8 @@ public: void setRenderProgress() { render_progress = true; } bool needRenderProgress() const { return render_progress; } - const FileTableEngineProgress & getFileTableEngineProgress() { return file_progress; } - void setFileTableEngineApproxBytesToProcess(size_t num_bytes) { file_progress.total_bytes_to_process = num_bytes; } + const FileProgress & getFileProgress() { return file_progress; } + void setFileTotalBytesToProcess(size_t num_bytes) { file_progress.total_bytes_to_process = num_bytes; } /** Set in executeQuery and InterpreterSelectQuery. Then it is used in IBlockInputStream, * to update and monitor information about the total number of resources spent for the query. diff --git a/src/Processors/Sources/SourceWithProgress.h b/src/Processors/Sources/SourceWithProgress.h index 37456eb970e..3aa7a81f418 100644 --- a/src/Processors/Sources/SourceWithProgress.h +++ b/src/Processors/Sources/SourceWithProgress.h @@ -52,24 +52,20 @@ public: void setLeafLimits(const SizeLimits & leaf_limits_) final {leaf_limits = leaf_limits_; } void setQuota(const std::shared_ptr & quota_) final { quota = quota_; } void setProcessListElement(QueryStatus * elem) final { process_list_elem = elem; } + void setProgressCallback(const ProgressCallback & callback) final { progress_callback = callback; } void addTotalRowsApprox(size_t value) final { total_rows_approx += value; } - /// This method might be overridden, if, during query execution, there is a Source, that needs - /// to add one more progress callback. - void setProgressCallback(const ProgressCallback & callback) override { progress_callback = callback; } - protected: /// Call this method to provide information about progress. void progress(const Progress & value); void work() override; - ProgressCallback progress_callback; - private: StreamLocalLimits limits; SizeLimits leaf_limits; std::shared_ptr quota; + ProgressCallback progress_callback; QueryStatus * process_list_elem = nullptr; /// The approximate total number of rows to read. For progress bar. diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 91fb3078b91..cb255701633 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -27,7 +27,6 @@ #include #include -#include #include #include @@ -39,8 +38,6 @@ #include #include #include -#include -#include namespace fs = std::filesystem; @@ -374,6 +371,13 @@ public: method = chooseCompressionMethod(current_path, storage->compression_method); } + /// For clickhouse-local add progress callback to display progress bar. + if (context->needRenderProgress() && context->getApplicationType() == Context::ApplicationType::LOCAL) + { + auto & in = static_cast(*nested_buffer); + in.setProgressCallback(context); + } + read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); auto get_block_for_format = [&]() -> Block @@ -432,86 +436,6 @@ public: } - void setProgressCallback(const ProgressCallback & callback) override - { - /// Add file progress callback only for clickhouse-local. - if (!context->needRenderProgress() || context->getApplicationType() != Context::ApplicationType::LOCAL) - { - progress_callback = callback; - return; - } - - auto file_progress_callback = [this](const Progress & progress) - { - static size_t increment = 0; - static const char * indicators[8] = - { - "\033[1;30m→\033[0m", - "\033[1;31m↘\033[0m", - "\033[1;32m↓\033[0m", - "\033[1;33m↙\033[0m", - "\033[1;34m←\033[0m", - "\033[1;35m↖\033[0m", - "\033[1;36m↑\033[0m", - "\033[1m↗\033[0m", - }; - size_t terminal_width = getTerminalWidth(); - - const auto & file_progress = context->getFileTableEngineProgress(); - WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); - - if (!file_progress.processed_bytes) - message << std::string(terminal_width, ' '); - - file_progress.processed_bytes += progress.read_bytes; - file_progress.processed_rows += progress.read_rows; - - /// Display progress bar only if .25 seconds have passed since query execution start. - size_t elapsed_ns = file_progress.watch.elapsed(); - if (elapsed_ns > 25000000 && progress.read_bytes > 0) - { - message << '\r'; - const char * indicator = indicators[increment % 8]; - size_t prefix_size = message.count(); - size_t processed_bytes = file_progress.processed_bytes.load(); - - message << indicator << " Progress: "; - message << formatReadableQuantity(file_progress.processed_rows) << " rows, "; - message << formatReadableSizeWithDecimalSuffix(file_progress.processed_bytes) << " bytes. "; - - size_t written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 1); /// Don't count invisible output (escape sequences). - ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); - - /// total_bytes_to_read is approximate, since its amount is taken as file size (or sum of all file sizes - /// from paths, generated for file table engine). And progress.read_bytes is counted accorging to columns types. - size_t total_bytes_corrected = std::max(processed_bytes, file_progress.total_bytes_to_process); - - if (width_of_progress_bar > 0) - { - std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); - message << "\033[0;32m" << bar << "\033[0m"; - - if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) - message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); - - message << ' ' << std::min((99 * file_progress.processed_bytes / file_progress.total_bytes_to_process), static_cast(99)) << '%'; - } - } - ++increment; - }; - - /// Progress callback can be added via context or via method in SourceWithProgress. - /// In executeQuery a callback from context is wrapped into another - /// progress callback and then passed to SourceWithProgress. Here another callback is - /// added to avoid overriding previous callbacks or avoid other callbacks overriding this one. - progress_callback = [callback, file_progress_callback](const Progress & progress) - { - callback(progress); - file_progress_callback(progress); - }; - } - - private: std::shared_ptr storage; StorageMetadataPtr metadata_snapshot; @@ -574,9 +498,9 @@ Pipe StorageFile::read( Pipes pipes; pipes.reserve(num_streams); - /// For clickhouse-local add progress callback to display in a progress bar. + /// For clickhouse-local to display progress bar. if (context->getApplicationType() == Context::ApplicationType::LOCAL) - context->setFileTableEngineApproxBytesToProcess(total_bytes_to_read); + context->setFileTotalBytesToProcess(total_bytes_to_read); for (size_t i = 0; i < num_streams; ++i) { diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index fc69f29cd20..fc4681a4305 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -85,9 +85,6 @@ protected: private: explicit StorageFile(CommonArguments args); - /// For clickhouse-local query display progress of processed files. - static void addProgressCallback(ContextPtr context); - std::string format_name; // We use format settings from global context + CREATE query for File table // function -- in this case, format_settings is set. @@ -110,7 +107,7 @@ private: Poco::Logger * log = &Poco::Logger::get("StorageFile"); - /// Approximate number of bytes to read. Needed for progress bar. + /// Total number of bytes to read (sums for multiple files in case of globs). Needed for progress bar. size_t total_bytes_to_read = 0; }; From 4bb63e1464a14fc2bba90d613fc8adfa60ba82aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Sun, 2 May 2021 00:27:46 +0800 Subject: [PATCH 042/491] add function bitpositionToArray --- src/Functions/FunctionsCoding.cpp | 1 + src/Functions/FunctionsCoding.h | 86 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/src/Functions/FunctionsCoding.cpp b/src/Functions/FunctionsCoding.cpp index c1e20a657b2..b40d6934ae8 100644 --- a/src/Functions/FunctionsCoding.cpp +++ b/src/Functions/FunctionsCoding.cpp @@ -23,6 +23,7 @@ void registerFunctionsCoding(FunctionFactory & factory) factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 933d4f01b92..80c79cb1a86 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1504,6 +1504,92 @@ public: } }; +class FunctionBitpositionToArray : public IFunction +{ +public: + static constexpr auto name = "bitpositionToArray"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 1; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isInteger(arguments[0])) + throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(arguments[0]); + } + + bool useDefaultImplementationForConstants() const override { return true; } + + template + bool tryExecute(const IColumn * column, ColumnPtr & out_column) const + { + using UnsignedT = make_unsigned_t; + + if (const ColumnVector * col_from = checkAndGetColumn>(column)) + { + auto col_values = ColumnVector::create(); + auto col_offsets = ColumnArray::ColumnOffsets::create(); + + typename ColumnVector::Container & res_values = col_values->getData(); + ColumnArray::Offsets & res_offsets = col_offsets->getData(); + + const typename ColumnVector::Container & vec_from = col_from->getData(); + size_t size = vec_from.size(); + res_offsets.resize(size); + res_values.reserve(size * 2); + + for (size_t row = 0; row < size; ++row) + { + UnsignedT x = vec_from[row]; + while (x) + { + UnsignedT y = x & (x - 1); + UnsignedT bit = x ^ y; + x = y; + res_values.push_back(std::log2(bit)); + } + res_offsets[row] = res_values.size(); + } + + out_column = ColumnArray::create(std::move(col_values), std::move(col_offsets)); + return true; + } + else + { + return false; + } + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const IColumn * in_column = arguments[0].column.get(); + ColumnPtr out_column; + + if (tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column) || + tryExecute(in_column, out_column)) + return out_column; + + throw Exception("Illegal column " + arguments[0].column->getName() + + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } +}; + class FunctionToStringCutToZero : public IFunction { public: From fa96613aeb12469f59c12fd93017a120aa81db13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Tue, 4 May 2021 23:46:00 +0800 Subject: [PATCH 043/491] update the way to get bitposition --- src/Functions/FunctionsCoding.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 80c79cb1a86..4b79a6beba6 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1550,12 +1550,15 @@ public: for (size_t row = 0; row < size; ++row) { UnsignedT x = vec_from[row]; + int position = 0; while (x) { - UnsignedT y = x & (x - 1); - UnsignedT bit = x ^ y; - x = y; - res_values.push_back(std::log2(bit)); + if(x & 1) + { + res_values.push_back(position); + } + x >>= 1 ; + position++; } res_offsets[row] = res_values.size(); } From 3c435bbcbdbe30c0ffd5c6711c8986bdde3874b6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 5 May 2021 10:37:10 +0000 Subject: [PATCH 044/491] Improve check for terminal size --- src/Common/TerminalSize.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Common/TerminalSize.cpp b/src/Common/TerminalSize.cpp index a020098aa44..304d875b02b 100644 --- a/src/Common/TerminalSize.cpp +++ b/src/Common/TerminalSize.cpp @@ -15,16 +15,19 @@ namespace DB::ErrorCodes uint16_t getTerminalWidth() { + struct winsize terminal_size {}; if (isatty(STDIN_FILENO)) { - struct winsize terminal_size {}; - if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); - - return terminal_size.ws_col; } - return 0; + else if (isatty(STDOUT_FILENO)) + { + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size)) + DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); + } + /// Default - 0. + return terminal_size.ws_col; } po::options_description createOptionsDescription(const std::string & caption, uint16_t terminal_width) From 2d99476cf15735205f9bb3d6b00a5d3fb957f1aa Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 5 May 2021 15:52:03 +0000 Subject: [PATCH 045/491] Use stderr instead of stdout --- src/Common/TerminalSize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/TerminalSize.cpp b/src/Common/TerminalSize.cpp index 304d875b02b..c53494fe9a0 100644 --- a/src/Common/TerminalSize.cpp +++ b/src/Common/TerminalSize.cpp @@ -21,9 +21,9 @@ uint16_t getTerminalWidth() if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); } - else if (isatty(STDOUT_FILENO)) + else if (isatty(STDERR_FILENO)) { - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size)) + if (ioctl(STDERR_FILENO, TIOCGWINSZ, &terminal_size)) DB::throwFromErrno("Cannot obtain terminal window size (ioctl TIOCGWINSZ)", DB::ErrorCodes::SYSTEM_ERROR); } /// Default - 0. From 41e576ceab12b79dbc1df23d6bbba723662d3f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Tue, 11 May 2021 12:18:11 +0800 Subject: [PATCH 046/491] add tests --- .../0_stateless/01866_bitposition.reference | 13 +++++++++++++ tests/queries/0_stateless/01866_bitposition.sql | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/queries/0_stateless/01866_bitposition.reference create mode 100644 tests/queries/0_stateless/01866_bitposition.sql diff --git a/tests/queries/0_stateless/01866_bitposition.reference b/tests/queries/0_stateless/01866_bitposition.reference new file mode 100644 index 00000000000..b31e2817f59 --- /dev/null +++ b/tests/queries/0_stateless/01866_bitposition.reference @@ -0,0 +1,13 @@ +[] +[0] +[0,1,2,3,4,5,6,7] +[0,1,2,3,4,5,6] +[7] +[] +[0] +[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] +[0,2,3,4,5,6,7,8,9,10,11,12,13,14] +[15] +[] +[0] +[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63] diff --git a/tests/queries/0_stateless/01866_bitposition.sql b/tests/queries/0_stateless/01866_bitposition.sql new file mode 100644 index 00000000000..d7ffa793c29 --- /dev/null +++ b/tests/queries/0_stateless/01866_bitposition.sql @@ -0,0 +1,16 @@ +select bitpositionToArray(0); +select bitpositionToArray(1); +select bitpositionToArray(-1); +select bitpositionToArray(127); +select bitpositionToArray(128); + +select bitpositionToArray(toInt16(0)); +select bitpositionToArray(toInt16(1)); +select bitpositionToArray(toInt16(-1)); +select bitpositionToArray(toInt16(32765)); +select bitpositionToArray(toInt16(32768)); + +select bitpositionToArray(toInt64(0)); +select bitpositionToArray(toInt64(1)); +select bitpositionToArray(toInt64(-1)); + From 5fe3c1ab6e89505a59fe144feff3ee8a7c2840c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Tue, 11 May 2021 12:39:03 +0800 Subject: [PATCH 047/491] add doc --- docs/en/sql-reference/functions/encoding-functions.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 6b72d3c2269..55c60de796c 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -172,3 +172,6 @@ Accepts an integer. Returns a string containing the list of powers of two that t Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order. +## bitpositionToArray(num) {#bitpositiontoarraynum} + +Accepts an integer. Returns an array of UInt64 numbers containing the list of positions of bit that equals 1. Numbers in the array are in ascending order. From 7ef770e9c688764e0cf99742f89a8815034a2d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Tue, 11 May 2021 16:55:23 +0800 Subject: [PATCH 048/491] fix style check --- src/Functions/FunctionsCoding.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 4b79a6beba6..f83e665928f 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1553,7 +1553,7 @@ public: int position = 0; while (x) { - if(x & 1) + if (x & 1) { res_values.push_back(position); } From dc6efeb8d1755aa0357ac4aa6d8f058cf5da24aa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 11 May 2021 12:19:43 +0300 Subject: [PATCH 049/491] Remove Russian admin tips as they are not updated (cherry picked from commit e85d87feec940cad155af8d6dbcdf2212a12eabe) --- docs/ru/operations/tips.md | 249 +------------------------------------ 1 file changed, 1 insertion(+), 248 deletions(-) mode change 100644 => 120000 docs/ru/operations/tips.md diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md deleted file mode 100644 index 4535767e8e0..00000000000 --- a/docs/ru/operations/tips.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -toc_priority: 58 -toc_title: "Советы по эксплуатации" ---- - -# Советы по эксплуатации {#sovety-po-ekspluatatsii} - -## CPU Scaling Governor {#cpu-scaling-governor} - -Всегда используйте `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе. - -``` bash -$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -``` - -## Ограничение CPU {#ogranichenie-cpu} - -Процессоры могут перегреваться. С помощью `dmesg` можно увидеть, если тактовая частота процессора была ограничена из-за перегрева. -Также ограничение может устанавливаться снаружи на уровне дата-центра. С помощью `turbostat` можно за этим наблюдать под нагрузкой. - -## Оперативная память {#operativnaia-pamiat} - -Для небольших объёмов данных (до ~200 Гб в сжатом виде) лучше всего использовать столько памяти не меньше, чем объём данных. -Для больших объёмов данных, при выполнении интерактивных (онлайн) запросов, стоит использовать разумный объём оперативной памяти (128 Гб или более) для того, чтобы горячее подмножество данных поместилось в кеше страниц. -Даже для объёмов данных в ~50 Тб на сервер, использование 128 Гб оперативной памяти намного лучше для производительности выполнения запросов, чем 64 Гб. - -Не выключайте overcommit. Значение `cat /proc/sys/vm/overcommit_memory` должно быть 0 or 1. Выполните: - -``` bash -$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory -``` - -## Huge Pages {#huge-pages} - -Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности. - -``` bash -$ echo 'madvise' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled -``` - -С помощью `perf top` можно наблюдать за временем, проведенном в ядре операционной системы для управления памятью. -Постоянные huge pages так же не нужно аллоцировать. - -## Подсистема хранения {#podsistema-khraneniia} - -Если ваш бюджет позволяет использовать SSD, используйте SSD. -В противном случае используйте HDD. SATA HDDs 7200 RPM подойдут. - -Предпочитайте много серверов с локальными жесткими дисками вместо меньшего числа серверов с подключенными дисковыми полками. -Но для хранения архивов с редкими запросами полки всё же подходят. - -## RAID {#raid} - -При использовании HDD можно объединить их RAID-10, RAID-5, RAID-6 или RAID-50. -Лучше использовать программный RAID в Linux (`mdadm`). Лучше не использовать LVM. -При создании RAID-10, нужно выбрать `far` расположение. -Если бюджет позволяет, лучше выбрать RAID-10. - -На более чем 4 дисках вместо RAID-5 нужно использовать RAID-6 (предпочтительнее) или RAID-50. -При использовании RAID-5, RAID-6 или RAID-50, нужно всегда увеличивать stripe_cache_size, так как значение по умолчанию выбрано не самым удачным образом. - -``` bash -$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size -``` - -Точное число стоит вычислять из числа устройств и размер блока по формуле: `2 * num_devices * chunk_size_in_bytes / 4096`. - -Размер блока в 1024 Кб подходит для всех конфигураций RAID. -Никогда не указывайте слишком маленький или слишком большой размер блока. - -На SSD можно использовать RAID-0. -Вне зависимости от использования RAID, всегда используйте репликацию для безопасности данных. - -Включите NCQ с длинной очередью. Для HDD стоит выбрать планировщик CFQ, а для SSD — noop. Не стоит уменьшать настройку readahead. -На HDD стоит включать кеш записи. - -## Файловая система {#failovaia-sistema} - -Ext4 самый проверенный вариант. Укажите опции монтирования `noatime,nobarrier`. -XFS также подходит, но не так тщательно протестирована в сочетании с ClickHouse. -Большинство других файловых систем также должны нормально работать. Файловые системы с отложенной аллокацией работают лучше. - -## Ядро Linux {#iadro-linux} - -Не используйте слишком старое ядро Linux. - -## Сеть {#set} - -При использовании IPv6, стоит увеличить размер кеша маршрутов. -Ядра Linux до 3.2 имели массу проблем в реализации IPv6. - -Предпочитайте как минимум 10 Гбит сеть. 1 Гбит также будет работать, но намного хуже для починки реплик с десятками терабайт данных или для обработки распределенных запросов с большим объёмом промежуточных данных. - -## ZooKeeper {#zookeeper} - -Вероятно вы уже используете ZooKeeper для других целей. Можно использовать ту же инсталляцию ZooKeeper, если она не сильно перегружена. - -Лучше использовать свежую версию ZooKeeper, как минимум 3.4.9. Версия в стабильных дистрибутивах Linux может быть устаревшей. - -Никогда не используете написанные вручную скрипты для переноса данных между разными ZooKeeper кластерами, потому что результат будет некорректный для sequential нод. Никогда не используйте утилиту «zkcopy», по той же причине: https://github.com/ksprojects/zkcopy/issues/15 - -Если вы хотите разделить существующий ZooKeeper кластер на два, правильный способ - увеличить количество его реплик, а затем переконфигурировать его как два независимых кластера. - -Не запускайте ZooKeeper на тех же серверах, что и ClickHouse. Потому что ZooKeeper очень чувствителен к задержкам, а ClickHouse может использовать все доступные системные ресурсы. - -С настройками по умолчанию, ZooKeeper является бомбой замедленного действия: - -> Сервер ZooKeeper не будет удалять файлы со старыми снепшоты и логами при использовании конфигурации по умолчанию (см. autopurge), это является ответственностью оператора. - -Эту бомбу нужно обезвредить. - -Далее описана конфигурация ZooKeeper (3.5.1), используемая в боевом окружении Яндекс.Метрики на момент 20 мая 2017 года: - -zoo.cfg: - -``` bash -# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=30000 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=10 - -maxClientCnxns=2000 - -maxSessionTimeout=60000000 -# the directory where the snapshot is stored. -dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/data -# Place the dataLogDir to a separate physical disc for better performance -dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/logs - -autopurge.snapRetainCount=10 -autopurge.purgeInterval=1 - - -# To avoid seeks ZooKeeper allocates space in the transaction log file in -# blocks of preAllocSize kilobytes. The default block size is 64M. One reason -# for changing the size of the blocks is to reduce the block size if snapshots -# are taken more often. (Also, see snapCount). -preAllocSize=131072 - -# Clients can submit requests faster than ZooKeeper can process them, -# especially if there are a lot of clients. To prevent ZooKeeper from running -# out of memory due to queued requests, ZooKeeper will throttle clients so that -# there is no more than globalOutstandingLimit outstanding requests in the -# system. The default limit is 1,000.ZooKeeper logs transactions to a -# transaction log. After snapCount transactions are written to a log file a -# snapshot is started and a new transaction log file is started. The default -# snapCount is 10,000. -snapCount=3000000 - -# If this option is defined, requests will be will logged to a trace file named -# traceFile.year.month.day. -#traceFile= - -# Leader accepts client connections. Default value is "yes". The leader machine -# coordinates updates. For higher update throughput at thes slight expense of -# read throughput the leader can be configured to not accept clients and focus -# on coordination. -leaderServes=yes - -standaloneEnabled=false -dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/zoo.cfg.dynamic -``` - -Версия Java: - -``` text -Java(TM) SE Runtime Environment (build 1.8.0_25-b17) -Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) -``` - -Параметры JVM: - -``` bash -NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} -ZOOCFGDIR=/etc/$NAME/conf - -# TODO this is really ugly -# How to find out, which jars are needed? -# seems, that log4j requires the log4j.properties file to be in the classpath -CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper" - -ZOOCFG="$ZOOCFGDIR/zoo.cfg" -ZOO_LOG_DIR=/var/log/$NAME -USER=zookeeper -GROUP=zookeeper -PIDDIR=/var/run/$NAME -PIDFILE=$PIDDIR/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -JAVA=/usr/bin/java -ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" -ZOO_LOG4J_PROP="INFO,ROLLINGFILE" -JMXLOCALONLY=false -JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '{{' }} '}}' }} \ - -Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '{{' }} '}}' }} \ - -Xloggc:/var/log/$NAME/zookeeper-gc.log \ - -XX:+UseGCLogFileRotation \ - -XX:NumberOfGCLogFiles=16 \ - -XX:GCLogFileSize=16M \ - -verbose:gc \ - -XX:+PrintGCTimeStamps \ - -XX:+PrintGCDateStamps \ - -XX:+PrintGCDetails - -XX:+PrintTenuringDistribution \ - -XX:+PrintGCApplicationStoppedTime \ - -XX:+PrintGCApplicationConcurrentTime \ - -XX:+PrintSafepointStatistics \ - -XX:+UseParNewGC \ - -XX:+UseConcMarkSweepGC \ --XX:+CMSParallelRemarkEnabled" -``` - -Salt init: - -``` text -description "zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} centralized coordination service" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 8192 8192 - -pre-start script - [ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment" ] || exit 0 - . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment - [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR - chown $USER:$GROUP $ZOO_LOG_DIR -end script - -script - . /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment - [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper - if [ -z "$JMXDISABLE" ]; then - JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY" - fi - exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} \ - -- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \ - -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG -end script -``` - diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md new file mode 120000 index 00000000000..9b3413bdbc3 --- /dev/null +++ b/docs/ru/operations/tips.md @@ -0,0 +1 @@ +../../en/operations/tips.md \ No newline at end of file From 7990c031687aa5b6fe92a067cae6ae968d3bfa0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Thu, 13 May 2021 14:26:12 +0800 Subject: [PATCH 050/491] add some examples contributes to use --- .../functions/encoding-functions.md | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 55c60de796c..ffd6b83b5ff 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -174,4 +174,48 @@ Accepts an integer. Returns an array of UInt64 numbers containing the list of po ## bitpositionToArray(num) {#bitpositiontoarraynum} -Accepts an integer. Returns an array of UInt64 numbers containing the list of positions of bit that equals 1. Numbers in the array are in ascending order. +Accepts an integer, argument will be convert to unsigned integer. Returns an array of UInt64 numbers containing the list of positions of bit that equals 1. Numbers in the array are in ascending order. + +**Syntax** + +```sql +bitpositionToArray(arg) +``` + +**Arguments** + +- `arg` — A value can be convert to unsigned integer .Types: [Int/UInt](../../sql-reference/data-types/int-uint.md) + +**Returned value** + +An array of UInt64 numbers containing the list of positions of bit that equals 1. Numbers in the array are in ascending order. + +**Example** + +Query: + +``` sql +select bitpositionToArray(toInt8(1)) as bitposition; +``` + +Result: + +``` text +┌─bitposition─┐ +│ [0] │ +└─────────────┘ +``` + +Query: + +``` sql +select bitpositionToArray(toInt8(-1)) as bitposition; +``` + +Result: + +``` text +┌─bitposition───────┐ +│ [0,1,2,3,4,5,6,7] │ +└───────────────────┘ +``` \ No newline at end of file From c25be65ebfb6679f80c563ba70403b2142758495 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 May 2021 22:56:42 +0000 Subject: [PATCH 051/491] Refactor progress bar, adjust progress indication for reading from file --- programs/client/Client.cpp | 80 +++++++------- programs/local/LocalServer.cpp | 32 +++--- programs/local/LocalServer.h | 11 +- src/Common/ProgressBar.h | 32 ------ ...ProgressBar.cpp => ProgressIndication.cpp} | 86 +++++++++++---- src/Common/ProgressIndication.h | 61 ++++++++++ src/IO/Progress.cpp | 77 +++++++++++++ src/IO/Progress.h | 104 +++++------------- src/IO/ReadBufferFromFileDescriptor.cpp | 69 +----------- src/Interpreters/Context.h | 12 +- src/Storages/StorageFile.cpp | 9 +- 11 files changed, 304 insertions(+), 269 deletions(-) delete mode 100644 src/Common/ProgressBar.h rename src/Common/{ProgressBar.cpp => ProgressIndication.cpp} (59%) create mode 100644 src/Common/ProgressIndication.h diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 898e1b92d09..99aef4f101f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -86,7 +85,7 @@ #include #include #include -#include +#include #if !defined(ARCADIA_BUILD) # include @@ -228,13 +227,13 @@ private: String server_version; String server_display_name; - Stopwatch watch; + /// true by default - for interactive mode, might be changed when --progress option is checked for + /// non-interactive mode. + bool need_render_progress = true; - /// The server periodically sends information about how much data was read since last time. - Progress progress; + bool written_first_block = false; - /// Progress bar - ProgressBar progress_bar; + ProgressIndication progress_indication; /// External tables info. std::list external_tables; @@ -534,7 +533,7 @@ private: if (!is_interactive) { - progress_bar.need_render_progress = config().getBool("progress", false); + need_render_progress = config().getBool("progress", false); echo_queries = config().getBool("echo", false); ignore_error = config().getBool("ignore-error", false); } @@ -1556,12 +1555,9 @@ private: } } - watch.restart(); processed_rows = 0; - progress.reset(); - progress_bar.show_progress_bar = false; - progress_bar.written_progress_chars = 0; - progress_bar.written_first_block = false; + written_first_block = false; + progress_indication.resetProgress(); { /// Temporarily apply query settings to context. @@ -1628,16 +1624,15 @@ private: if (is_interactive) { - std::cout << std::endl << processed_rows << " rows in set. Elapsed: " << watch.elapsedSeconds() << " sec. "; - - if (progress.read_rows >= 1000) - writeFinalProgress(); + std::cout << std::endl << processed_rows << " rows in set. Elapsed: " << progress_indication.elapsedSeconds() << " sec. "; + /// Write final progress if it makes sence to do so. + writeFinalProgress(); std::cout << std::endl << std::endl; } else if (print_time_to_stderr) { - std::cerr << watch.elapsedSeconds() << "\n"; + std::cerr << progress_indication.elapsedSeconds() << "\n"; } } @@ -1813,7 +1808,17 @@ private: try { if (need_render_progress) + { + /// Set total_bytes_to_read for current fd. + FileProgress file_progress(0, std_in.size()); + progress_indication.updateProgress(Progress(file_progress)); + + /// Set callback to be called on file progress. + progress_indication.setFileProgressCallback(context, true); + + /// Add callback to track reading from fd. std_in.setProgressCallback(context); + } sendDataFrom(std_in, sample, columns_description); } @@ -1937,7 +1942,7 @@ private: cancelled = true; if (is_interactive) { - progress_bar.clearProgress(); + progress_indication.clearProgressOutput(); std::cout << "Cancelling query." << std::endl; } @@ -2164,7 +2169,7 @@ private: current_format = "Vertical"; /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. - if (!progress_bar.need_render_progress) + if (!need_render_progress) block_out_stream = context->getOutputStreamParallelIfPossible(current_format, *out_buf, block); else block_out_stream = context->getOutputStream(current_format, *out_buf, block); @@ -2223,25 +2228,25 @@ private: if (block.rows() == 0 || (query_fuzzer_runs != 0 && processed_rows >= 100)) return; - if (progress_bar.need_render_progress) - progress_bar.clearProgress(); + if (need_render_progress) + progress_indication.clearProgressOutput(); block_out_stream->write(block); - progress_bar.written_first_block = true; + written_first_block = true; /// Received data block is immediately displayed to the user. block_out_stream->flush(); /// Restore progress bar after data block. - if (progress_bar.need_render_progress) - progress_bar.writeProgress(progress, watch.elapsed()); + if (need_render_progress) + progress_indication.writeProgress(); } void onLogData(Block & block) { initLogsOutputStream(); - progress_bar.clearProgress(); + progress_indication.clearProgressOutput(); logs_out_stream->write(block); logs_out_stream->flush(); } @@ -2262,28 +2267,23 @@ private: void onProgress(const Progress & value) { - if (!progress_bar.updateProgress(progress, value)) + if (!progress_indication.updateProgress(value)) { // Just a keep-alive update. return; } + if (block_out_stream) block_out_stream->onProgress(value); - progress_bar.writeProgress(progress, watch.elapsed()); + + if (need_render_progress) + progress_indication.writeProgress(); } void writeFinalProgress() { - std::cout << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, " - << formatReadableSizeWithDecimalSuffix(progress.read_bytes); - - size_t elapsed_ns = watch.elapsed(); - if (elapsed_ns) - std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " - << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; - else - std::cout << ". "; + progress_indication.writeFinalProgress(); } @@ -2304,7 +2304,7 @@ private: void onEndOfStream() { - progress_bar.clearProgress(); + progress_indication.clearProgressOutput(); if (block_out_stream) block_out_stream->writeSuffix(); @@ -2314,9 +2314,9 @@ private: resetOutput(); - if (is_interactive && !progress_bar.written_first_block) + if (is_interactive && !written_first_block) { - progress_bar.clearProgress(); + progress_indication.clearProgressOutput(); std::cout << "Ok." << std::endl; } } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 93ab9e6e995..c53230eaf3a 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -389,34 +389,29 @@ void LocalServer::processQueries() CurrentThread::QueryScope query_scope_holder(context); ///Set progress show - progress_bar.need_render_progress = config().getBool("progress", false); + need_render_progress = config().getBool("progress", false); - if (progress_bar.need_render_progress) + if (need_render_progress) { context->setProgressCallback([&](const Progress & value) - { - if (!progress_bar.updateProgress(progress, value)) - { - // Just a keep-alive update. - return; - } - progress_bar.writeProgress(progress, watch.elapsed()); - }); + { + /// Write progress only if progress was updated + if (progress_indication.updateProgress(value)) + progress_indication.writeProgress(); + }); } bool echo_queries = config().hasOption("echo") || config().hasOption("verbose"); - if (config().hasOption("progress")) - context->setRenderProgress(); + + if (need_render_progress) + progress_indication.setFileProgressCallback(context); + std::exception_ptr exception; for (const auto & query : queries) { - watch.restart(); - progress.reset(); - progress_bar.show_progress_bar = false; - progress_bar.written_progress_chars = 0; - progress_bar.written_first_block = false; - + written_first_block = false; + progress_indication.resetProgress(); ReadBufferFromString read_buf(query); WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO); @@ -566,7 +561,6 @@ void LocalServer::init(int argc, char ** argv) ("output-format", po::value(), "default output format") ("stacktrace", "print stack traces of exceptions") - ("progress", "show progress for File table engine") ("echo", "print query before execution") ("verbose", "print query and other debugging info") ("logger.console", po::value()->implicit_value(true), "Log to console") diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index c5e9d5716dd..a9e4668617e 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include namespace DB { @@ -49,9 +49,12 @@ protected: /// Settings specified via command line args Settings cmd_settings; - ProgressBar progress_bar; - Progress progress; - Stopwatch watch; + + bool need_render_progress = false; + + bool written_first_block = false; + + ProgressIndication progress_indication; std::optional temporary_directory_to_delete; }; diff --git a/src/Common/ProgressBar.h b/src/Common/ProgressBar.h deleted file mode 100644 index 895cea51f6a..00000000000 --- a/src/Common/ProgressBar.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include - -/// http://en.wikipedia.org/wiki/ANSI_escape_code -#define CLEAR_TO_END_OF_LINE "\033[K" - -namespace DB -{ - -struct ProgressBar -{ -public: - - static bool updateProgress(Progress & progress, const Progress & value); - void writeProgress(const Progress & progress, const size_t elapsed_ns); - void clearProgress(); - - /// For interactive mode always show progress bar, for non-interactive mode it is accessed from config(). - bool need_render_progress = true; - - bool show_progress_bar = false; - - size_t written_progress_chars = 0; - - bool written_first_block = false; - - bool clear_progress = false; -}; - -} diff --git a/src/Common/ProgressBar.cpp b/src/Common/ProgressIndication.cpp similarity index 59% rename from src/Common/ProgressBar.cpp rename to src/Common/ProgressIndication.cpp index 27e5eca1ea5..34f6530d59a 100644 --- a/src/Common/ProgressBar.cpp +++ b/src/Common/ProgressIndication.cpp @@ -1,22 +1,64 @@ -#include "ProgressBar.h" +#include "ProgressIndication.h" #include #include #include #include + namespace DB { -bool ProgressBar::updateProgress(Progress & progress, const Progress & value) +bool ProgressIndication::updateProgress(const Progress & value) { return progress.incrementPiecewiseAtomically(value); } -void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ns) +void ProgressIndication::clearProgressOutput() { - if (!need_render_progress) + if (written_progress_chars) + { + written_progress_chars = 0; + std::cerr << "\r" CLEAR_TO_END_OF_LINE; + } +} + +void ProgressIndication::resetProgress() +{ + watch.restart(); + progress.reset(); + show_progress_bar = false; + written_progress_chars = 0; +} + +void ProgressIndication::setFileProgressCallback(ContextPtr context, bool write_progress_on_update) +{ + context->setFileProgressCallback([&](const FileProgress & file_progress) + { + progress.incrementPiecewiseAtomically(Progress(file_progress)); + + if (write_progress_on_update) + writeProgress(); + }); +} + +void ProgressIndication::writeFinalProgress() +{ + if (progress.read_rows < 1000) return; + std::cout << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, " + << formatReadableSizeWithDecimalSuffix(progress.read_bytes); + + auto elapsed_ns = watch.elapsed(); + if (elapsed_ns) + std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " + << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; + else + std::cout << ". "; +} + +void ProgressIndication::writeProgress() +{ /// Output all progress bar commands to stderr at once to avoid flicker. WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); @@ -45,26 +87,37 @@ void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ message << '\r'; size_t prefix_size = message.count(); + size_t read_bytes = progress.read_raw_bytes ? progress.read_raw_bytes : progress.read_bytes; message << indicator << " Progress: "; - message << formatReadableQuantity(progress.read_rows) << " rows, " - << formatReadableSizeWithDecimalSuffix(progress.read_bytes); + << formatReadableSizeWithDecimalSuffix(read_bytes); + auto elapsed_ns = watch.elapsed(); if (elapsed_ns) message << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " - << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; + << formatReadableSizeWithDecimalSuffix(read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; else message << ". "; written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 2); /// Don't count invisible output (escape sequences). /// If the approximate number of rows to process is known, we can display a progress bar and percentage. - if (progress.total_rows_to_read > 0) + if (progress.total_rows_to_read || progress.total_raw_bytes_to_read) { - size_t total_rows_corrected = std::max(progress.read_rows, progress.total_rows_to_read); + size_t current_count, max_count; + if (progress.total_rows_to_read) + { + current_count = progress.read_rows; + max_count = std::max(progress.read_rows, progress.total_rows_to_read); + } + else + { + current_count = progress.read_raw_bytes; + max_count = std::max(progress.read_raw_bytes, progress.total_raw_bytes_to_read); + } /// To avoid flicker, display progress bar only if .5 seconds have passed since query execution start /// and the query is less than halfway done. @@ -72,7 +125,7 @@ void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ if (elapsed_ns > 500000000) { /// Trigger to start displaying progress bar. If query is mostly done, don't display it. - if (progress.read_rows * 2 < total_rows_corrected) + if (current_count * 2 < max_count) show_progress_bar = true; if (show_progress_bar) @@ -81,7 +134,7 @@ void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ if (width_of_progress_bar > 0) { std::string bar - = UnicodeBar::render(UnicodeBar::getWidth(progress.read_rows, 0, total_rows_corrected, width_of_progress_bar)); + = UnicodeBar::render(UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar)); message << "\033[0;32m" << bar << "\033[0m"; if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); @@ -90,7 +143,7 @@ void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ } /// Underestimate percentage a bit to avoid displaying 100%. - message << ' ' << (99 * progress.read_rows / total_rows_corrected) << '%'; + message << ' ' << (99 * current_count / max_count) << '%'; } message << CLEAR_TO_END_OF_LINE; @@ -99,13 +152,4 @@ void ProgressBar::writeProgress(const Progress & progress, const size_t elapsed_ message.next(); } -void ProgressBar::clearProgress() -{ - if (written_progress_chars) - { - written_progress_chars = 0; - std::cerr << "\r" CLEAR_TO_END_OF_LINE; - } -} - } diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h new file mode 100644 index 00000000000..0567488cf15 --- /dev/null +++ b/src/Common/ProgressIndication.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include + + +/// http://en.wikipedia.org/wiki/ANSI_escape_code +#define CLEAR_TO_END_OF_LINE "\033[K" + +namespace DB +{ + +class ProgressIndication +{ +public: + /// Write progress to stderr. + void writeProgress(); + + void writeFinalProgress(); + + /// Clear stderr output. + void clearProgressOutput(); + + /// Reset progress values. + void resetProgress(); + + /// Update Progress object. It can be updated from: + /// 1. onProgress in clickhouse-client; + /// 2. ProgressCallback via setProgressCallback methrod in: + /// - context (used in clickhouse-local, can also be added in arbitrary place) + /// - SourceWithProgress (also in streams) + /// - readBufferFromFileDescriptor (for file processing progress) + bool updateProgress(const Progress & value); + + /// In some cases there is a need to update progress value, when there is no access to progress_inidcation object. + /// In this case it is added via context. + /// `write_progress_on_update` is needed to write progress for loading files data via pipe in non-interactive mode. + void setFileProgressCallback(ContextPtr context, bool write_progress_on_update = false); + + /// How much seconds passed since query execution start. + UInt64 elapsedSeconds() const { return watch.elapsedSeconds(); } + +private: + /// Show progress bar only if on first writeProgress call the query is not yet half done. + bool show_progress_bar = false; + + /// Width of how much has been printed currently into stderr. Used to define size of progress bar and + /// to check whether progress output needs to be cleared. + size_t written_progress_chars = 0; + + /// The server periodically sends information about how much data was read since last time. + /// This information is stored here. + Progress progress; + + /// Track query execution time. + Stopwatch watch; +}; + +} diff --git a/src/IO/Progress.cpp b/src/IO/Progress.cpp index f255a39955b..7cc1504a978 100644 --- a/src/IO/Progress.cpp +++ b/src/IO/Progress.cpp @@ -63,6 +63,83 @@ void ProgressValues::writeJSON(WriteBuffer & out) const writeCString("\"}", out); } +bool Progress::incrementPiecewiseAtomically(const Progress & rhs) +{ + read_rows += rhs.read_rows; + read_bytes += rhs.read_bytes; + read_raw_bytes += rhs.read_raw_bytes; + + total_rows_to_read += rhs.total_rows_to_read; + total_raw_bytes_to_read += rhs.total_raw_bytes_to_read; + + written_rows += rhs.written_rows; + written_bytes += rhs.written_bytes; + + return rhs.read_rows || rhs.written_rows; +} + +void Progress::reset() +{ + read_rows = 0; + read_bytes = 0; + read_raw_bytes = 0; + + total_rows_to_read = 0; + total_raw_bytes_to_read = 0; + + written_rows = 0; + written_bytes = 0; +} + +ProgressValues Progress::getValues() const +{ + ProgressValues res; + + res.read_rows = read_rows.load(std::memory_order_relaxed); + res.read_bytes = read_bytes.load(std::memory_order_relaxed); + res.read_raw_bytes = read_raw_bytes.load(std::memory_order_relaxed); + + res.total_rows_to_read = total_rows_to_read.load(std::memory_order_relaxed); + res.total_raw_bytes_to_read = total_raw_bytes_to_read.load(std::memory_order_relaxed); + + res.written_rows = written_rows.load(std::memory_order_relaxed); + res.written_bytes = written_bytes.load(std::memory_order_relaxed); + + return res; +} + +ProgressValues Progress::fetchAndResetPiecewiseAtomically() +{ + ProgressValues res; + + res.read_rows = read_rows.fetch_and(0); + res.read_bytes = read_bytes.fetch_and(0); + res.read_raw_bytes = read_raw_bytes.fetch_and(0); + + res.total_rows_to_read = total_rows_to_read.fetch_and(0); + res.total_raw_bytes_to_read = total_raw_bytes_to_read.fetch_and(0); + + res.written_rows = written_rows.fetch_and(0); + res.written_bytes = written_bytes.fetch_and(0); + + return res; +} + +Progress & Progress::operator=(Progress && other) +{ + read_rows = other.read_rows.load(std::memory_order_relaxed); + read_bytes = other.read_bytes.load(std::memory_order_relaxed); + read_raw_bytes = other.read_raw_bytes.load(std::memory_order_relaxed); + + total_rows_to_read = other.total_rows_to_read.load(std::memory_order_relaxed); + total_raw_bytes_to_read = other.total_raw_bytes_to_read.load(std::memory_order_relaxed); + + written_rows = other.written_rows.load(std::memory_order_relaxed); + written_bytes = other.written_bytes.load(std::memory_order_relaxed); + + return *this; +} + void Progress::read(ReadBuffer & in, UInt64 server_revision) { ProgressValues values; diff --git a/src/IO/Progress.h b/src/IO/Progress.h index 240564849e3..446acef9abd 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -19,7 +19,11 @@ struct ProgressValues { size_t read_rows; size_t read_bytes; + size_t read_raw_bytes; + size_t total_rows_to_read; + size_t total_raw_bytes_to_read; + size_t written_rows; size_t written_bytes; @@ -47,32 +51,13 @@ struct WriteProgress : written_rows(written_rows_), written_bytes(written_bytes_) {} }; -/// Track progress of processing file base buffer. -/// Used to display progress of loading data from stdin, for file table engine, etc. struct FileProgress { - /// Track elapsed time. - Stopwatch watch; - size_t total_bytes_to_process; + /// Here read_bytes (raw bytes) - do not equal ReadProgress::read_bytes, which are calculated according to column types. + size_t read_bytes; + size_t total_bytes_to_read; - /// FileProgress lies in Context and accessed via const reference. - /// These fields are allowed to be updated in a progress callback. - mutable std::atomic processed_bytes; - - FileProgress() : total_bytes_to_process(0), processed_bytes(0) {} - - FileProgress(const FileProgress & other) - : watch(other.watch) - , total_bytes_to_process(other.total_bytes_to_process) - , processed_bytes(other.processed_bytes.load()) {} - - FileProgress & operator=(FileProgress other) - { - watch = other.watch; - total_bytes_to_process = other.total_bytes_to_process; - processed_bytes = other.processed_bytes.load(); - return *this; - } + FileProgress(size_t read_bytes_, size_t total_bytes_to_read_ = 0) : read_bytes(read_bytes_), total_bytes_to_read(total_bytes_to_read_) {} }; @@ -84,87 +69,50 @@ struct Progress { std::atomic read_rows {0}; /// Rows (source) processed. std::atomic read_bytes {0}; /// Bytes (uncompressed, source) processed. + std::atomic read_raw_bytes {0}; /// Raw bytes processed. - /** How much rows must be processed, in total, approximately. Non-zero value is sent when there is information about some new part of job. - * Received values must be summed to get estimate of total rows to process. + /** How much rows/bytes must be processed, in total, approximately. Non-zero value is sent when there is information about + * some new part of job. Received values must be summed to get estimate of total rows to process. + * `total_raw_bytes_to_process` is used for file table engine or when reading from file descriptor. * Used for rendering progress bar on client. */ std::atomic total_rows_to_read {0}; - + std::atomic total_raw_bytes_to_read {0}; std::atomic written_rows {0}; std::atomic written_bytes {0}; Progress() = default; + Progress(size_t read_rows_, size_t read_bytes_, size_t total_rows_to_read_ = 0) : read_rows(read_rows_), read_bytes(read_bytes_), total_rows_to_read(total_rows_to_read_) {} + explicit Progress(ReadProgress read_progress) : read_rows(read_progress.read_rows), read_bytes(read_progress.read_bytes), total_rows_to_read(read_progress.total_rows_to_read) {} + explicit Progress(WriteProgress write_progress) : written_rows(write_progress.written_rows), written_bytes(write_progress.written_bytes) {} + explicit Progress(FileProgress file_progress) + : read_raw_bytes(file_progress.read_bytes), total_raw_bytes_to_read(file_progress.total_bytes_to_read) {} + void read(ReadBuffer & in, UInt64 server_revision); + void write(WriteBuffer & out, UInt64 client_revision) const; + /// Progress in JSON format (single line, without whitespaces) is used in HTTP headers. void writeJSON(WriteBuffer & out) const; /// Each value separately is changed atomically (but not whole object). - bool incrementPiecewiseAtomically(const Progress & rhs) - { - read_rows += rhs.read_rows; - read_bytes += rhs.read_bytes; - total_rows_to_read += rhs.total_rows_to_read; - written_rows += rhs.written_rows; - written_bytes += rhs.written_bytes; + bool incrementPiecewiseAtomically(const Progress & rhs); - return rhs.read_rows || rhs.written_rows; - } + void reset(); - void reset() - { - read_rows = 0; - read_bytes = 0; - total_rows_to_read = 0; - written_rows = 0; - written_bytes = 0; - } + ProgressValues getValues() const; - ProgressValues getValues() const - { - ProgressValues res; + ProgressValues fetchAndResetPiecewiseAtomically(); - res.read_rows = read_rows.load(std::memory_order_relaxed); - res.read_bytes = read_bytes.load(std::memory_order_relaxed); - res.total_rows_to_read = total_rows_to_read.load(std::memory_order_relaxed); - res.written_rows = written_rows.load(std::memory_order_relaxed); - res.written_bytes = written_bytes.load(std::memory_order_relaxed); - - return res; - } - - ProgressValues fetchAndResetPiecewiseAtomically() - { - ProgressValues res; - - res.read_rows = read_rows.fetch_and(0); - res.read_bytes = read_bytes.fetch_and(0); - res.total_rows_to_read = total_rows_to_read.fetch_and(0); - res.written_rows = written_rows.fetch_and(0); - res.written_bytes = written_bytes.fetch_and(0); - - return res; - } - - Progress & operator=(Progress && other) - { - read_rows = other.read_rows.load(std::memory_order_relaxed); - read_bytes = other.read_bytes.load(std::memory_order_relaxed); - total_rows_to_read = other.total_rows_to_read.load(std::memory_order_relaxed); - written_rows = other.written_rows.load(std::memory_order_relaxed); - written_bytes = other.written_bytes.load(std::memory_order_relaxed); - - return *this; - } + Progress & operator=(Progress && other); Progress(Progress && other) { diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index 786d944c0ae..babdc953514 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -13,8 +13,6 @@ #include #include -#define CLEAR_TO_END_OF_LINE "\033[K" - namespace ProfileEvents { @@ -191,71 +189,14 @@ off_t ReadBufferFromFileDescriptor::size() void ReadBufferFromFileDescriptor::setProgressCallback(ContextPtr context) { - /// Keep file progress and total bytes to process in context and not in readBuffer, because - /// multiple files might share the same progress (for example, for file table engine when globs are used) - /// and total_bytes_to_process will contain sum of sizes of all files. + auto file_progress_callback = context->getFileProgressCallback(); - if (!context->getFileProgress().total_bytes_to_process) - context->setFileTotalBytesToProcess(size()); + if (!file_progress_callback) + return; - setProfileCallback([context](const ProfileInfo & progress) + setProfileCallback([file_progress_callback](const ProfileInfo & progress) { - static size_t increment = 0; - static const char * indicators[8] = - { - "\033[1;30m→\033[0m", - "\033[1;31m↘\033[0m", - "\033[1;32m↓\033[0m", - "\033[1;33m↙\033[0m", - "\033[1;34m←\033[0m", - "\033[1;35m↖\033[0m", - "\033[1;36m↑\033[0m", - "\033[1m↗\033[0m", - }; - size_t terminal_width = getTerminalWidth(); - WriteBufferFromFileDescriptor message(STDERR_FILENO, 1024); - - const auto & file_progress = context->getFileProgress(); - - if (!file_progress.processed_bytes) - message << std::string(terminal_width, ' '); - message << '\r'; - file_progress.processed_bytes += progress.bytes_read; - - const char * indicator = indicators[increment % 8]; - size_t prefix_size = message.count(); - size_t processed_bytes = file_progress.processed_bytes.load(); - - message << indicator << " Progress: "; - message << formatReadableSizeWithDecimalSuffix(file_progress.processed_bytes); - message << " from " << formatReadableSizeWithDecimalSuffix(file_progress.total_bytes_to_process) << " bytes. "; - - /// Display progress bar only if .25 seconds have passed since query execution start. - size_t elapsed_ns = file_progress.watch.elapsed(); - if (elapsed_ns > 25000000) - { - size_t written_progress_chars = message.count() - prefix_size - (strlen(indicator) - 1); /// Don't count invisible output (escape sequences). - ssize_t width_of_progress_bar = static_cast(terminal_width) - written_progress_chars - strlen(" 99%"); - - size_t total_bytes_corrected = std::max(processed_bytes, file_progress.total_bytes_to_process); - - if (width_of_progress_bar > 0 && progress.bytes_read > 0) - { - std::string bar = UnicodeBar::render(UnicodeBar::getWidth(processed_bytes, 0, total_bytes_corrected, width_of_progress_bar)); - message << "\033[0;32m" << bar << "\033[0m"; - - if (width_of_progress_bar > static_cast(bar.size() / UNICODE_BAR_CHAR_SIZE)) - message << std::string(width_of_progress_bar - bar.size() / UNICODE_BAR_CHAR_SIZE, ' '); - } - - /// Underestimate percentage a bit to avoid displaying 100%. - message << ' ' << (99 * file_progress.processed_bytes / file_progress.total_bytes_to_process) << '%'; - } - - message << CLEAR_TO_END_OF_LINE; - message.next(); - - ++increment; + file_progress_callback(FileProgress(progress.bytes_read, 0)); }); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index b6e6175e313..f5c8533daaf 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -181,8 +181,9 @@ private: using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. - FileProgress file_progress; /// Progress data to track processing of file based buffer(s). - bool render_progress = false; + + using FileProgressCallback = std::function; + FileProgressCallback file_progress_callback; /// Callback for tracking progress of file loading. QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. StorageID insertion_table = StorageID::createEmpty(); /// Saved insertion table in query context @@ -588,11 +589,8 @@ public: /// Used in InterpreterSelectQuery to pass it to the IBlockInputStream. ProgressCallback getProgressCallback() const; - void setRenderProgress() { render_progress = true; } - bool needRenderProgress() const { return render_progress; } - - const FileProgress & getFileProgress() { return file_progress; } - void setFileTotalBytesToProcess(size_t num_bytes) { file_progress.total_bytes_to_process = num_bytes; } + void setFileProgressCallback(FileProgressCallback && callback) { file_progress_callback = callback; } + FileProgressCallback getFileProgressCallback() const { return file_progress_callback; } /** Set in executeQuery and InterpreterSelectQuery. Then it is used in IBlockInputStream, * to update and monitor information about the total number of resources spent for the query. diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index cb255701633..bb20a9972aa 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -372,7 +372,7 @@ public: } /// For clickhouse-local add progress callback to display progress bar. - if (context->needRenderProgress() && context->getApplicationType() == Context::ApplicationType::LOCAL) + if (context->getApplicationType() == Context::ApplicationType::LOCAL) { auto & in = static_cast(*nested_buffer); in.setProgressCallback(context); @@ -498,9 +498,10 @@ Pipe StorageFile::read( Pipes pipes; pipes.reserve(num_streams); - /// For clickhouse-local to display progress bar. - if (context->getApplicationType() == Context::ApplicationType::LOCAL) - context->setFileTotalBytesToProcess(total_bytes_to_read); + /// Set total number of bytes to process. For progress bar. + auto progress_callback = context->getFileProgressCallback(); + if (context->getApplicationType() == Context::ApplicationType::LOCAL && progress_callback) + progress_callback(FileProgress(0, total_bytes_to_read)); for (size_t i = 0; i < num_streams; ++i) { From 4574c090483a91dfdd1004e8c012c25e7b598286 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 May 2021 08:35:51 +0000 Subject: [PATCH 052/491] Fix style check --- programs/client/Client.cpp | 2 +- src/Common/ProgressIndication.cpp | 7 ++++++- src/Common/ProgressIndication.h | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 99aef4f101f..4e59016138d 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1625,7 +1625,7 @@ private: if (is_interactive) { std::cout << std::endl << processed_rows << " rows in set. Elapsed: " << progress_indication.elapsedSeconds() << " sec. "; - /// Write final progress if it makes sence to do so. + /// Write final progress if it makes sense to do so. writeFinalProgress(); std::cout << std::endl << std::endl; diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 34f6530d59a..cabe823f062 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -4,6 +4,9 @@ #include #include +/// FIXME: progress bar in clickhouse-local needs to be cleared after query execution +/// - same as it is now in clickhouse-client. Also there is no writeFinalProgress call +/// in clickhouse-local. namespace DB { @@ -28,10 +31,12 @@ void ProgressIndication::resetProgress() progress.reset(); show_progress_bar = false; written_progress_chars = 0; + write_progress_on_update = false; } -void ProgressIndication::setFileProgressCallback(ContextPtr context, bool write_progress_on_update) +void ProgressIndication::setFileProgressCallback(ContextPtr context, bool write_progress_on_update_) { + write_progress_on_update = write_progress_on_update_; context->setFileProgressCallback([&](const FileProgress & file_progress) { progress.incrementPiecewiseAtomically(Progress(file_progress)); diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 0567488cf15..df161387ec3 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -56,6 +55,8 @@ private: /// Track query execution time. Stopwatch watch; + + bool write_progress_on_update = false; }; } From 35bbbd05b6f9d47b16a6fe017290295bf22d6008 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 12 May 2021 18:06:25 +0300 Subject: [PATCH 053/491] Try to not remove lowcard in merge join --- src/Columns/ColumnLowCardinality.cpp | 23 +++++++++++++ src/Columns/ColumnLowCardinality.h | 3 ++ src/Columns/ColumnUnique.h | 5 +-- src/Interpreters/MergeJoin.cpp | 51 ++++++++++++++++------------ 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index c4d7e75dd2d..23bda370908 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -307,6 +307,29 @@ int ColumnLowCardinality::compareAtWithCollation(size_t n, size_t m, const IColu return compareAtImpl(n, m, rhs, nan_direction_hint, &collator); } +int ColumnLowCardinality::compareAtGeneric(size_t n, size_t m, const IColumn & lhs, const IColumn & rhs, int nan_direction_hint) +{ + const auto * left_lc = typeid_cast(&lhs); + const auto * right_lc = typeid_cast(&rhs); + + if (left_lc && right_lc) + return left_lc->compareAt(n, m, rhs, nan_direction_hint); + + if (left_lc) + { + size_t n_lc_index = left_lc->getIndexes().getUInt(n); + return left_lc->getDictionary().compareAt(n_lc_index, m, rhs, nan_direction_hint); + } + + if (right_lc) + { + size_t m_lc_index = right_lc->getIndexes().getUInt(n); + return -right_lc->getDictionary().compareAt(n, m_lc_index, lhs, -nan_direction_hint); + } + + throw DB::Exception("One column should have low cardinality type", ErrorCodes::LOGICAL_ERROR); +} + void ColumnLowCardinality::compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index fc607021ccf..662e2403e70 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -122,6 +122,9 @@ public: int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; + /// Like compareAt, but allows one of columns to be non-low cardinality of the same type + static int compareAtGeneric(size_t n, size_t m, const IColumn & lhs, const IColumn & rhs, int nan_direction_hint); + void compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 51b45be53fd..242e19655b6 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -407,8 +407,9 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } } - const auto & column_unique = static_cast(rhs); - return getNestedColumn()->compareAt(n, m, *column_unique.getNestedColumn(), nan_direction_hint); + if (const auto * column_unique = typeid_cast(&rhs)) + return getNestedColumn()->compareAt(n, m, *column_unique->getNestedColumn(), nan_direction_hint); + return getNestedColumn()->compareAt(n, m, rhs, nan_direction_hint); } template diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index a9f50cdda0e..67151cadc40 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ namespace ErrorCodes namespace { -template +template int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos) { static constexpr int null_direction_hint = 1; @@ -73,6 +74,14 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, } } + if constexpr (has_diff_lowcard) + { + if (left_column.lowCardinality() != right_column.lowCardinality()) + { + return ColumnLowCardinality::compareAtGeneric(lhs_pos, rhs_pos, left_column, right_column, null_direction_hint); + } + } + return left_column.compareAt(lhs_pos, rhs_pos, right_column, null_direction_hint); } @@ -192,27 +201,30 @@ public: bool atEnd() const { return impl.getRow() >= impl.rows; } void nextN(size_t num) { impl.getPosRef() += num; } - void setCompareNullability(const MergeJoinCursor & rhs) + void setCompareProperties(const MergeJoinCursor & rhs) { - has_left_nullable = false; - has_right_nullable = false; + has_left_nullable = has_right_nullable = has_diff_lowcard = false; for (size_t i = 0; i < impl.sort_columns_size; ++i) { has_left_nullable = has_left_nullable || isColumnNullable(*impl.sort_columns[i]); has_right_nullable = has_right_nullable || isColumnNullable(*rhs.impl.sort_columns[i]); + has_diff_lowcard = has_diff_lowcard || impl.sort_columns[i]->lowCardinality() != rhs.impl.sort_columns[i]->lowCardinality(); } } Range getNextEqualRange(MergeJoinCursor & rhs) { if (has_left_nullable && has_right_nullable) - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); else if (has_left_nullable) - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); else if (has_right_nullable) - return getNextEqualRangeImpl(rhs); - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); + else if (has_diff_lowcard) + return getNextEqualRangeImpl(rhs); + + return getNextEqualRangeImpl(rhs); } int intersect(const Block & min_max, const Names & key_names) @@ -230,10 +242,10 @@ public: const auto & right_column = *min_max.getByName(key_names[i]).column; /// cannot get by position cause of possible duplicates if (!first_vs_max) - first_vs_max = nullableCompareAt(left_column, right_column, position(), 1); + first_vs_max = nullableCompareAt(left_column, right_column, position(), 1); if (!last_vs_min) - last_vs_min = nullableCompareAt(left_column, right_column, last_position, 0); + last_vs_min = nullableCompareAt(left_column, right_column, last_position, 0); } if (first_vs_max > 0) @@ -247,13 +259,14 @@ private: SortCursorImpl impl; bool has_left_nullable = false; bool has_right_nullable = false; + bool has_diff_lowcard = false; - template + template Range getNextEqualRangeImpl(MergeJoinCursor & rhs) { while (!atEnd() && !rhs.atEnd()) { - int cmp = compareAtCursor(rhs); + int cmp = compareAtCursor(rhs); if (cmp < 0) impl.next(); else if (cmp > 0) @@ -265,7 +278,7 @@ private: return Range{impl.getRow(), rhs.impl.getRow(), 0, 0}; } - template + template int ALWAYS_INLINE compareAtCursor(const MergeJoinCursor & rhs) const { for (size_t i = 0; i < impl.sort_columns_size; ++i) @@ -273,7 +286,7 @@ private: const auto * left_column = impl.sort_columns[i]; const auto * right_column = rhs.impl.sort_columns[i]; - int res = nullableCompareAt(*left_column, *right_column, impl.getRow(), rhs.impl.getRow()); + int res = nullableCompareAt(*left_column, *right_column, impl.getRow(), rhs.impl.getRow()); if (res) return res; } @@ -465,8 +478,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right } table_join->splitAdditionalColumns(right_sample_block, right_table_keys, right_columns_to_add); - JoinCommon::removeLowCardinalityInplace(right_table_keys); - JoinCommon::removeLowCardinalityInplace(right_sample_block, table_join->keyNamesRight()); const NameSet required_right_keys = table_join->requiredRightKeys(); for (const auto & column : right_table_keys) @@ -593,7 +604,6 @@ bool MergeJoin::saveRightBlock(Block && block) Block MergeJoin::modifyRightBlock(const Block & src_block) const { Block block = materializeBlock(src_block); - JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesRight()); return block; } @@ -611,7 +621,6 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); - JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft(), false); sortBlock(block, left_sort_description); @@ -760,7 +769,7 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareNullability(right_cursor); + left_cursor.setCompareProperties(right_cursor); /// Set right cursor position in first continuation right block if constexpr (is_all) @@ -821,7 +830,7 @@ bool MergeJoin::allInnerJoin(MergeJoinCursor & left_cursor, const Block & left_b { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareNullability(right_cursor); + left_cursor.setCompareProperties(right_cursor); /// Set right cursor position in first continuation right block right_cursor.nextN(right_block_info.skip); @@ -863,7 +872,7 @@ bool MergeJoin::semiLeftJoin(MergeJoinCursor & left_cursor, const Block & left_b { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareNullability(right_cursor); + left_cursor.setCompareProperties(right_cursor); while (!left_cursor.atEnd() && !right_cursor.atEnd()) { From be966f0e8c40341b0cf16ed058d829b9637df0a9 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 15 May 2021 14:26:59 +0300 Subject: [PATCH 054/491] Revert "Try to not remove lowcard in merge join" This reverts commit 35bbbd05b6f9d47b16a6fe017290295bf22d6008. --- src/Columns/ColumnLowCardinality.cpp | 23 ------------- src/Columns/ColumnLowCardinality.h | 3 -- src/Columns/ColumnUnique.h | 5 ++- src/Interpreters/MergeJoin.cpp | 51 ++++++++++++---------------- 4 files changed, 23 insertions(+), 59 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index 23bda370908..c4d7e75dd2d 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -307,29 +307,6 @@ int ColumnLowCardinality::compareAtWithCollation(size_t n, size_t m, const IColu return compareAtImpl(n, m, rhs, nan_direction_hint, &collator); } -int ColumnLowCardinality::compareAtGeneric(size_t n, size_t m, const IColumn & lhs, const IColumn & rhs, int nan_direction_hint) -{ - const auto * left_lc = typeid_cast(&lhs); - const auto * right_lc = typeid_cast(&rhs); - - if (left_lc && right_lc) - return left_lc->compareAt(n, m, rhs, nan_direction_hint); - - if (left_lc) - { - size_t n_lc_index = left_lc->getIndexes().getUInt(n); - return left_lc->getDictionary().compareAt(n_lc_index, m, rhs, nan_direction_hint); - } - - if (right_lc) - { - size_t m_lc_index = right_lc->getIndexes().getUInt(n); - return -right_lc->getDictionary().compareAt(n, m_lc_index, lhs, -nan_direction_hint); - } - - throw DB::Exception("One column should have low cardinality type", ErrorCodes::LOGICAL_ERROR); -} - void ColumnLowCardinality::compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 662e2403e70..fc607021ccf 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -122,9 +122,6 @@ public: int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; - /// Like compareAt, but allows one of columns to be non-low cardinality of the same type - static int compareAtGeneric(size_t n, size_t m, const IColumn & lhs, const IColumn & rhs, int nan_direction_hint); - void compareColumn(const IColumn & rhs, size_t rhs_row_num, PaddedPODArray * row_indexes, PaddedPODArray & compare_results, int direction, int nan_direction_hint) const override; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 242e19655b6..51b45be53fd 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -407,9 +407,8 @@ int ColumnUnique::compareAt(size_t n, size_t m, const IColumn & rhs, } } - if (const auto * column_unique = typeid_cast(&rhs)) - return getNestedColumn()->compareAt(n, m, *column_unique->getNestedColumn(), nan_direction_hint); - return getNestedColumn()->compareAt(n, m, rhs, nan_direction_hint); + const auto & column_unique = static_cast(rhs); + return getNestedColumn()->compareAt(n, m, *column_unique.getNestedColumn(), nan_direction_hint); } template diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 67151cadc40..a9f50cdda0e 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,7 @@ namespace ErrorCodes namespace { -template +template int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos) { static constexpr int null_direction_hint = 1; @@ -74,14 +73,6 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, } } - if constexpr (has_diff_lowcard) - { - if (left_column.lowCardinality() != right_column.lowCardinality()) - { - return ColumnLowCardinality::compareAtGeneric(lhs_pos, rhs_pos, left_column, right_column, null_direction_hint); - } - } - return left_column.compareAt(lhs_pos, rhs_pos, right_column, null_direction_hint); } @@ -201,30 +192,27 @@ public: bool atEnd() const { return impl.getRow() >= impl.rows; } void nextN(size_t num) { impl.getPosRef() += num; } - void setCompareProperties(const MergeJoinCursor & rhs) + void setCompareNullability(const MergeJoinCursor & rhs) { - has_left_nullable = has_right_nullable = has_diff_lowcard = false; + has_left_nullable = false; + has_right_nullable = false; for (size_t i = 0; i < impl.sort_columns_size; ++i) { has_left_nullable = has_left_nullable || isColumnNullable(*impl.sort_columns[i]); has_right_nullable = has_right_nullable || isColumnNullable(*rhs.impl.sort_columns[i]); - has_diff_lowcard = has_diff_lowcard || impl.sort_columns[i]->lowCardinality() != rhs.impl.sort_columns[i]->lowCardinality(); } } Range getNextEqualRange(MergeJoinCursor & rhs) { if (has_left_nullable && has_right_nullable) - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); else if (has_left_nullable) - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); else if (has_right_nullable) - return getNextEqualRangeImpl(rhs); - else if (has_diff_lowcard) - return getNextEqualRangeImpl(rhs); - - return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); + return getNextEqualRangeImpl(rhs); } int intersect(const Block & min_max, const Names & key_names) @@ -242,10 +230,10 @@ public: const auto & right_column = *min_max.getByName(key_names[i]).column; /// cannot get by position cause of possible duplicates if (!first_vs_max) - first_vs_max = nullableCompareAt(left_column, right_column, position(), 1); + first_vs_max = nullableCompareAt(left_column, right_column, position(), 1); if (!last_vs_min) - last_vs_min = nullableCompareAt(left_column, right_column, last_position, 0); + last_vs_min = nullableCompareAt(left_column, right_column, last_position, 0); } if (first_vs_max > 0) @@ -259,14 +247,13 @@ private: SortCursorImpl impl; bool has_left_nullable = false; bool has_right_nullable = false; - bool has_diff_lowcard = false; - template + template Range getNextEqualRangeImpl(MergeJoinCursor & rhs) { while (!atEnd() && !rhs.atEnd()) { - int cmp = compareAtCursor(rhs); + int cmp = compareAtCursor(rhs); if (cmp < 0) impl.next(); else if (cmp > 0) @@ -278,7 +265,7 @@ private: return Range{impl.getRow(), rhs.impl.getRow(), 0, 0}; } - template + template int ALWAYS_INLINE compareAtCursor(const MergeJoinCursor & rhs) const { for (size_t i = 0; i < impl.sort_columns_size; ++i) @@ -286,7 +273,7 @@ private: const auto * left_column = impl.sort_columns[i]; const auto * right_column = rhs.impl.sort_columns[i]; - int res = nullableCompareAt(*left_column, *right_column, impl.getRow(), rhs.impl.getRow()); + int res = nullableCompareAt(*left_column, *right_column, impl.getRow(), rhs.impl.getRow()); if (res) return res; } @@ -478,6 +465,8 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right } table_join->splitAdditionalColumns(right_sample_block, right_table_keys, right_columns_to_add); + JoinCommon::removeLowCardinalityInplace(right_table_keys); + JoinCommon::removeLowCardinalityInplace(right_sample_block, table_join->keyNamesRight()); const NameSet required_right_keys = table_join->requiredRightKeys(); for (const auto & column : right_table_keys) @@ -604,6 +593,7 @@ bool MergeJoin::saveRightBlock(Block && block) Block MergeJoin::modifyRightBlock(const Block & src_block) const { Block block = materializeBlock(src_block); + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesRight()); return block; } @@ -621,6 +611,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft(), false); sortBlock(block, left_sort_description); @@ -769,7 +760,7 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareProperties(right_cursor); + left_cursor.setCompareNullability(right_cursor); /// Set right cursor position in first continuation right block if constexpr (is_all) @@ -830,7 +821,7 @@ bool MergeJoin::allInnerJoin(MergeJoinCursor & left_cursor, const Block & left_b { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareProperties(right_cursor); + left_cursor.setCompareNullability(right_cursor); /// Set right cursor position in first continuation right block right_cursor.nextN(right_block_info.skip); @@ -872,7 +863,7 @@ bool MergeJoin::semiLeftJoin(MergeJoinCursor & left_cursor, const Block & left_b { const Block & right_block = *right_block_info.block; MergeJoinCursor right_cursor(right_block, right_merge_description); - left_cursor.setCompareProperties(right_cursor); + left_cursor.setCompareNullability(right_cursor); while (!left_cursor.atEnd() && !right_cursor.atEnd()) { From 9091bba3d66a6c2c21a922af3b23b9d112bc7f5a Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 15 May 2021 14:39:13 +0300 Subject: [PATCH 055/491] Restore correct cardinality in HashJoin --- src/Functions/materialize.h | 2 ++ src/Interpreters/HashJoin.cpp | 14 +++++++++++--- src/Interpreters/join_common.cpp | 15 +++++++++++++++ src/Interpreters/join_common.h | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 741669e69d9..c08f4c21c9b 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -1,6 +1,8 @@ #pragma once #include #include +#include +#include namespace DB { diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 51373ea49d4..251553436c7 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -160,7 +160,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, { if (nullable) { - JoinCommon::convertColumnToNullable(column, true); + JoinCommon::convertColumnToNullable(column, false); if (column.type->isNullable() && !negative_null_map.empty()) { MutableColumnPtr mutable_column = IColumn::mutate(std::move(column.column)); @@ -1081,7 +1081,11 @@ void HashJoin::joinBlockImpl( const auto & col = block.getByName(left_name); bool is_nullable = nullable_right_side || right_key.type->isNullable(); - block.insert(correctNullability({col.column, col.type, right_key.name}, is_nullable)); + + ColumnWithTypeAndName right_col(col.column, col.type, right_key.name); + if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) + JoinCommon::changeLowCardinalityInplace(right_col); + block.insert(correctNullability(std::move(right_col), is_nullable)); } } else if (has_required_right_keys) @@ -1106,7 +1110,11 @@ void HashJoin::joinBlockImpl( bool is_nullable = nullable_right_side || right_key.type->isNullable(); ColumnPtr thin_column = filterWithBlanks(col.column, filter); - block.insert(correctNullability({thin_column, col.type, right_key.name}, is_nullable, null_map_filter)); + + ColumnWithTypeAndName right_col(thin_column, col.type, right_key.name); + if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) + JoinCommon::changeLowCardinalityInplace(right_col); + block.insert(correctNullability(std::move(right_col), is_nullable, null_map_filter)); if constexpr (need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 80299610a44..c61ad62e95e 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -49,6 +49,21 @@ ColumnPtr changeLowCardinality(const ColumnPtr & column, const ColumnPtr & dst_s namespace JoinCommon { +void changeLowCardinalityInplace(ColumnWithTypeAndName & column) +{ + if (column.type->lowCardinality()) + { + column.type = recursiveRemoveLowCardinality(column.type); + column.column = column.column->convertToFullColumnIfLowCardinality(); + } + else + { + column.type = std::make_shared(column.type); + MutableColumnPtr lc = column.type->createColumn(); + typeid_cast(*lc).insertRangeFromFullColumn(*column.column, 0, column.column->size()); + column.column = std::move(lc); + } +} bool canBecomeNullable(const DataTypePtr & type) { diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 9a000aa107a..9334b9d672f 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -41,6 +41,8 @@ void addDefaultValues(IColumn & column, const DataTypePtr & type, size_t count); bool typesEqualUpToNullability(DataTypePtr left_type, DataTypePtr right_type); +void changeLowCardinalityInplace(ColumnWithTypeAndName & column); + } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. From 8dddcebe8ce1988409a6e66ffa624e052d6ffb54 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 15 May 2021 17:09:51 +0300 Subject: [PATCH 056/491] Handle correct nullability with low card in HashJoin --- src/Columns/ColumnLowCardinality.h | 1 + src/Columns/ColumnUnique.h | 9 +++++++++ src/Columns/IColumnUnique.h | 1 + src/Interpreters/HashJoin.cpp | 8 +++++--- src/Interpreters/join_common.cpp | 14 ++++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index fc607021ccf..4af2cb2f36e 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -188,6 +188,7 @@ public: */ bool nestedIsNullable() const { return isColumnNullable(*dictionary.getColumnUnique().getNestedColumn()); } void nestedToNullable() { dictionary.getColumnUnique().nestedToNullable(); } + void nestedRemoveNullable() { dictionary.getColumnUnique().nestedRemoveNullable(); } const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); } const ColumnPtr & getDictionaryPtr() const { return dictionary.getColumnUniquePtr(); } diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 51b45be53fd..c138962cf29 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -51,6 +51,7 @@ public: const ColumnPtr & getNestedNotNullableColumn() const override { return column_holder; } bool nestedColumnIsNullable() const override { return is_nullable; } void nestedToNullable() override; + void nestedRemoveNullable() override; size_t uniqueInsert(const Field & x) override; size_t uniqueInsertFrom(const IColumn & src, size_t n) override; @@ -271,6 +272,14 @@ void ColumnUnique::nestedToNullable() createNullMask(); } +template +void ColumnUnique::nestedRemoveNullable() +{ + is_nullable = false; + nested_null_mask = nullptr; + nested_column_nullable = nullptr; +} + template const ColumnPtr & ColumnUnique::getNestedColumn() const { diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 2c1c542fce5..5e6473219d1 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -24,6 +24,7 @@ public: virtual bool nestedColumnIsNullable() const = 0; virtual void nestedToNullable() = 0; + virtual void nestedRemoveNullable() = 0; /// Returns array with StringRefHash calculated for each row of getNestedNotNullableColumn() column. /// Returns nullptr if nested column doesn't contain strings. Otherwise calculates hash (if it wasn't). diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 251553436c7..2af136e7350 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -160,7 +160,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, { if (nullable) { - JoinCommon::convertColumnToNullable(column, false); + JoinCommon::convertColumnToNullable(column); if (column.type->isNullable() && !negative_null_map.empty()) { MutableColumnPtr mutable_column = IColumn::mutate(std::move(column.column)); @@ -1085,7 +1085,8 @@ void HashJoin::joinBlockImpl( ColumnWithTypeAndName right_col(col.column, col.type, right_key.name); if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); - block.insert(correctNullability(std::move(right_col), is_nullable)); + right_col = correctNullability(std::move(right_col), is_nullable); + block.insert(right_col); } } else if (has_required_right_keys) @@ -1114,7 +1115,8 @@ void HashJoin::joinBlockImpl( ColumnWithTypeAndName right_col(thin_column, col.type, right_key.name); if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); - block.insert(correctNullability(std::move(right_col), is_nullable, null_map_filter)); + right_col = correctNullability(std::move(right_col), is_nullable, null_map_filter); + block.insert(right_col); if constexpr (need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index c61ad62e95e..29ed2b01c3e 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -122,6 +122,20 @@ void convertColumnsToNullable(Block & block, size_t starting_pos) /// @warning It assumes that every NULL has default value in nested column (or it does not matter) void removeColumnNullability(ColumnWithTypeAndName & column) { + if (column.type->lowCardinality()) + { + /// LowCardinality(Nullable(T)) case + ColumnLowCardinality * col_as_lc = assert_cast(column.column->assumeMutable().get()); + if (col_as_lc->nestedIsNullable()) + { + col_as_lc->nestedRemoveNullable(); + const auto & dict_type = typeid_cast(column.type.get())->getDictionaryType(); + column.type = std::make_shared(removeNullable(dict_type)); + } + + return; + } + if (!column.type->isNullable()) return; From 9ba31eacc69e4f9d32f9dd5eaaf30ea0accc0ae6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 15 May 2021 17:10:13 +0300 Subject: [PATCH 057/491] Upd 01049_join_low_card_bug --- .../01049_join_low_card_bug.reference | 231 +++++++++++++----- .../0_stateless/01049_join_low_card_bug.sql | 174 ++++++------- 2 files changed, 266 insertions(+), 139 deletions(-) diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.reference b/tests/queries/0_stateless/01049_join_low_card_bug.reference index b4ed8176652..d3610bb97cf 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.reference +++ b/tests/queries/0_stateless/01049_join_low_card_bug.reference @@ -1,80 +1,201 @@ - LowCardinality(String) str LowCardinality(String) LowCardinality(String) +str_r LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - LowCardinality(String) str LowCardinality(String) LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) String - str LowCardinality(String) String - str LowCardinality(String) String - str LowCardinality(String) String - String +str_l LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) LowCardinality(String) LowCardinality(String) str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) LowCardinality(String) LowCardinality(String) str_l str_l +str LowCardinality(String) + LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str LowCardinality(String) + LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String str str LowCardinality(String) LowCardinality(String) str str +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String LowCardinality(String) LowCardinality(String) str_l str_l +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String LowCardinality(String) LowCardinality(String) str_l str_l str String String +str_r String str String - str String LowCardinality(String) - str String LowCardinality(String) - str String LowCardinality(String) - str String LowCardinality(String) - LowCardinality(String) +str String + String +str_l String +str_r String +str String +str_l String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) String String str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) String String str_l str_l str LowCardinality(String) LowCardinality(String) +str_r LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) -\N Nullable(String) +str LowCardinality(String) + LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) \N \N LowCardinality(String) LowCardinality(String) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) \N \N LowCardinality(String) LowCardinality(String) str_l str_l str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) +str_r Nullable(String) +str Nullable(String) \N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) Nullable(String) Nullable(String) str_l str_l +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) Nullable(String) Nullable(String) str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +-- join_use_nulls -- +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +LowCardinality(String) LowCardinality(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(String) LowCardinality(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +String String str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +String String str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +String String str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +String String str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N LowCardinality(Nullable(String)) +str_r Nullable(String) +str Nullable(String) +\N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N str LowCardinality(Nullable(String)) \N LowCardinality(Nullable(String)) str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N Nullable(String) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) +str_r Nullable(String) +str Nullable(String) +\N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.sql b/tests/queries/0_stateless/01049_join_low_card_bug.sql index 07558770abf..39427c962fa 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.sql +++ b/tests/queries/0_stateless/01049_join_low_card_bug.sql @@ -12,129 +12,135 @@ CREATE TABLE nr (x Nullable(UInt32), lc Nullable(String)) ENGINE = Memory; CREATE TABLE l_lc (x UInt32, lc LowCardinality(String)) ENGINE = Memory; CREATE TABLE r_lc (x UInt32, lc LowCardinality(String)) ENGINE = Memory; -INSERT INTO r VALUES (0, 'str'); -INSERT INTO nr VALUES (0, 'str'); -INSERT INTO r_lc VALUES (0, 'str'); +INSERT INTO r VALUES (0, 'str'), (1, 'str_r'); +INSERT INTO nr VALUES (0, 'str'), (1, 'str_r'); +INSERT INTO r_lc VALUES (0, 'str'), (1, 'str_r'); + +INSERT INTO l VALUES (0, 'str'), (2, 'str_l'); +INSERT INTO nl VALUES (0, 'str'), (2, 'str_l'); +INSERT INTO l_lc VALUES (0, 'str'), (2, 'str_l'); -- -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc); - -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(r.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(r.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(r.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(r.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l RIGHT JOIN r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l RIGHT JOIN r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l FULL JOIN r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l FULL JOIN r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc); -SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc); +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l FULL JOIN r_lc AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x); -SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x); -SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l RIGHT JOIN nr AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l FULL JOIN nr AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM l_lc AS l FULL JOIN nr AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x); -SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc); -SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x); -SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc); +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM nl AS l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM nl AS l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc), toTypeName(materialize(r.lc)) FROM nl AS l FULL JOIN r_lc AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +SELECT '-- join_use_nulls --'; SET join_use_nulls = 1; -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc); -SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x); -SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc); +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l FULL JOIN r_lc AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x); -SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc); -SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x); -SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc); +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x; -- -SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x); -SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc); -SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x); -SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc); +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc) ORDER BY x; -SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x); -SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc); +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; DROP TABLE l; DROP TABLE r; From 80466427ad33ae893b83dd5efa8fd149e152a0d0 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 17 May 2021 14:18:03 +0300 Subject: [PATCH 058/491] Correct cardinality in MergeJoin (same as in input) --- src/Interpreters/MergeJoin.cpp | 25 ++- src/Interpreters/MergeJoin.h | 2 + .../01049_join_low_card_bug.reference | 201 ++++++++++++++++++ .../0_stateless/01049_join_low_card_bug.sql | 129 +++++++++++ 4 files changed, 354 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index a9f50cdda0e..26463c8c6ed 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -464,6 +464,12 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right ErrorCodes::PARAMETER_OUT_OF_BOUND); } + for (const auto & right_key : table_join->keyNamesRight()) + { + if (right_sample_block.getByName(right_key).type->lowCardinality()) + lowcard_right_keys.push_back(right_key); + } + table_join->splitAdditionalColumns(right_sample_block, right_table_keys, right_columns_to_add); JoinCommon::removeLowCardinalityInplace(right_table_keys); JoinCommon::removeLowCardinalityInplace(right_sample_block, table_join->keyNamesRight()); @@ -607,10 +613,18 @@ bool MergeJoin::addJoinedBlock(const Block & src_block, bool) void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { + Names lowcard_keys = lowcard_right_keys; if (block) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); + + for (const auto & column_name : table_join->keyNamesLeft()) + { + if (block.getByName(column_name).type->lowCardinality()) + lowcard_keys.push_back(column_name); + } + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft(), false); sortBlock(block, left_sort_description); @@ -646,14 +660,20 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) if (!not_processed && left_blocks_buffer) not_processed = std::make_shared(NotProcessed{{}, 0, 0, 0}); + for (const auto & column_name : lowcard_keys) + { + if (!block.has(column_name)) + continue; + if (auto & col = block.getByName(column_name); !col.type->lowCardinality()) + JoinCommon::changeLowCardinalityInplace(col); + } + JoinCommon::restoreLowCardinalityInplace(block); } template void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) { - //std::shared_lock lock(rwlock); - size_t rows_to_reserve = is_left ? block.rows() : 0; MutableColumns left_columns = makeMutableColumns(block, (is_all ? rows_to_reserve : 0)); MutableColumns right_columns = makeMutableColumns(right_columns_to_add, rows_to_reserve); @@ -702,7 +722,6 @@ void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) left_cursor.nextN(left_key_tail); joinInequalsLeft(block, left_columns, right_columns_to_add, right_columns, left_cursor.position(), left_cursor.end()); - //left_cursor.nextN(left_cursor.end() - left_cursor.position()); changeLeftColumns(block, std::move(left_columns)); addRightColumns(block, std::move(right_columns)); diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index f286e74b385..b6bde8fb131 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -103,6 +103,8 @@ private: const size_t max_rows_in_right_block; const size_t max_files_to_merge; + Names lowcard_right_keys; + void changeLeftColumns(Block & block, MutableColumns && columns) const; void addRightColumns(Block & block, MutableColumns && columns); diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.reference b/tests/queries/0_stateless/01049_join_low_card_bug.reference index d3610bb97cf..c6de3c202d7 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.reference +++ b/tests/queries/0_stateless/01049_join_low_card_bug.reference @@ -199,3 +199,204 @@ LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +str LowCardinality(String) + LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str LowCardinality(String) + LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) LowCardinality(String) LowCardinality(String) str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(String) LowCardinality(String) +LowCardinality(String) LowCardinality(String) str str LowCardinality(String) LowCardinality(String) str str +LowCardinality(String) LowCardinality(String) LowCardinality(String) LowCardinality(String) str_l str_l +str LowCardinality(String) + LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str LowCardinality(String) + LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String str str LowCardinality(String) LowCardinality(String) str str +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String LowCardinality(String) LowCardinality(String) str_l str_l +String String str_r str_r LowCardinality(String) LowCardinality(String) +String String str str LowCardinality(String) LowCardinality(String) str str +String String LowCardinality(String) LowCardinality(String) str_l str_l +str String + String +str_r String +str String +str String + String +str_l String +str_r String +str String +str_l String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) String String str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r String String +LowCardinality(String) LowCardinality(String) str str String String str str +LowCardinality(String) LowCardinality(String) String String str_l str_l +str LowCardinality(String) + LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str LowCardinality(String) + LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) \N \N LowCardinality(String) LowCardinality(String) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(String) LowCardinality(String) +Nullable(String) Nullable(String) str str LowCardinality(String) LowCardinality(String) str str +Nullable(String) Nullable(String) \N \N LowCardinality(String) LowCardinality(String) str_l str_l +str Nullable(String) +\N Nullable(String) +str Nullable(String) +str_r Nullable(String) +str Nullable(String) +\N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) Nullable(String) Nullable(String) str_l str_l +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) Nullable(String) Nullable(String) str_l str_l +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +-- join_use_nulls -- +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +LowCardinality(String) LowCardinality(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(String) LowCardinality(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(String) LowCardinality(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +String String str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +String String str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +String String str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +String String str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +str Nullable(String) +\N Nullable(String) +str Nullable(String) +str_r Nullable(String) +str Nullable(String) +\N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str str LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str +Nullable(String) Nullable(String) \N \N LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_l str_l +Nullable(String) Nullable(String) str_r str_r LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N +str Nullable(String) +\N Nullable(String) +str Nullable(String) +str_r Nullable(String) +str Nullable(String) +\N Nullable(String) +str_l Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(String) LowCardinality(String) str str Nullable(String) Nullable(String) str str +LowCardinality(String) LowCardinality(String) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str str Nullable(String) Nullable(String) str str +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) \N \N Nullable(String) Nullable(String) str_l str_l +LowCardinality(Nullable(String)) LowCardinality(Nullable(String)) str_r str_r Nullable(String) Nullable(String) \N \N diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.sql b/tests/queries/0_stateless/01049_join_low_card_bug.sql index 39427c962fa..ba245c2eba6 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.sql +++ b/tests/queries/0_stateless/01049_join_low_card_bug.sql @@ -20,6 +20,135 @@ INSERT INTO l VALUES (0, 'str'), (2, 'str_l'); INSERT INTO nl VALUES (0, 'str'), (2, 'str_l'); INSERT INTO l_lc VALUES (0, 'str'), (2, 'str_l'); + +SET join_use_nulls = 0; + +-- + +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +SELECT '-- join_use_nulls --'; + +SET join_use_nulls = 1; + +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l RIGHT JOIN r USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l FULL JOIN r USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc RIGHT JOIN nr USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM l_lc FULL JOIN nr USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +-- + +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl RIGHT JOIN r_lc USING (lc) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (x) ORDER BY x; +SELECT lc, toTypeName(lc) FROM nl FULL JOIN r_lc USING (lc) ORDER BY x; + +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x) ORDER BY x; +SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x; + +set join_algorithm = 'partial_merge'; + +SET join_use_nulls = 0; + -- SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x; From 392148ba866686189f36ae0864d417011d1aadcd Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 17 May 2021 14:24:34 +0300 Subject: [PATCH 059/491] Handle incorrect type of column in JoinCommon::removeColumnNullability --- src/Interpreters/join_common.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 29ed2b01c3e..def19b4d7a8 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -125,13 +125,12 @@ void removeColumnNullability(ColumnWithTypeAndName & column) if (column.type->lowCardinality()) { /// LowCardinality(Nullable(T)) case - ColumnLowCardinality * col_as_lc = assert_cast(column.column->assumeMutable().get()); - if (col_as_lc->nestedIsNullable()) - { + const auto & dict_type = typeid_cast(column.type.get())->getDictionaryType(); + column.type = std::make_shared(removeNullable(dict_type)); + + ColumnLowCardinality * col_as_lc = typeid_cast(column.column->assumeMutable().get()); + if (col_as_lc && col_as_lc->nestedIsNullable()) col_as_lc->nestedRemoveNullable(); - const auto & dict_type = typeid_cast(column.type.get())->getDictionaryType(); - column.type = std::make_shared(removeNullable(dict_type)); - } return; } From 627b1556227a03142e9b009ae3b060f5ea16dbdf Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 19 May 2021 17:54:53 +0300 Subject: [PATCH 060/491] Rename 01049_join_low_card_bug -> 01049_join_low_card_bug_long --- ..._card_bug.reference => 01049_join_low_card_bug_long.reference} | 0 ...049_join_low_card_bug.sql => 01049_join_low_card_bug_long.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01049_join_low_card_bug.reference => 01049_join_low_card_bug_long.reference} (100%) rename tests/queries/0_stateless/{01049_join_low_card_bug.sql => 01049_join_low_card_bug_long.sql} (100%) diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.reference b/tests/queries/0_stateless/01049_join_low_card_bug_long.reference similarity index 100% rename from tests/queries/0_stateless/01049_join_low_card_bug.reference rename to tests/queries/0_stateless/01049_join_low_card_bug_long.reference diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.sql b/tests/queries/0_stateless/01049_join_low_card_bug_long.sql similarity index 100% rename from tests/queries/0_stateless/01049_join_low_card_bug.sql rename to tests/queries/0_stateless/01049_join_low_card_bug_long.sql From 3e6d3d4ecb47047e4633828d4daa3875a1bcae1b Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 22 May 2021 08:47:19 +0000 Subject: [PATCH 061/491] Postgres schema for insert --- contrib/libpqxx | 2 +- .../fetchPostgreSQLTableStructure.cpp | 2 +- src/Storages/StoragePostgreSQL.cpp | 59 ++++++++++--------- .../test_storage_postgresql/test.py | 5 ++ 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/contrib/libpqxx b/contrib/libpqxx index 58d2a028d16..8fcd332202e 160000 --- a/contrib/libpqxx +++ b/contrib/libpqxx @@ -1 +1 @@ -Subproject commit 58d2a028d1600225ac3a478d6b3a06ba2f0c01f6 +Subproject commit 8fcd332202e806d3d82ca27c60c8f7ebb6282aed diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 2eb10bf273f..a310315dcc8 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -116,7 +116,7 @@ std::shared_ptr fetchPostgreSQLTableStructure( try { pqxx::read_transaction tx(connection_holder->get()); - pqxx::stream_from stream(tx, pqxx::from_query, std::string_view(query)); + auto stream{pqxx::stream_from::query(tx, query)}; std::tuple row; while (stream >> row) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 1d863a43b29..11f81e2f910 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -100,34 +100,29 @@ public: explicit PostgreSQLBlockOutputStream( const StorageMetadataPtr & metadata_snapshot_, postgres::ConnectionHolderPtr connection_holder_, - const std::string & remote_table_name_) + const String & remote_table_name_, + const String & remote_table_schema_) : metadata_snapshot(metadata_snapshot_) , connection_holder(std::move(connection_holder_)) , remote_table_name(remote_table_name_) + , remote_table_schema(remote_table_schema_) { } Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } - - void writePrefix() override - { - work = std::make_unique(connection_holder->get()); - } - - void write(const Block & block) override { - if (!work) - return; + if (!inserter) + inserter = std::make_unique(connection_holder->get(), + remote_table_schema.empty() ? pqxx::table_path({remote_table_name}) + : pqxx::table_path({remote_table_schema, remote_table_name}), + block.getNames()); const auto columns = block.getColumns(); const size_t num_rows = block.rows(), num_cols = block.columns(); const auto data_types = block.getDataTypes(); - if (!stream_inserter) - stream_inserter = std::make_unique(*work, remote_table_name, block.getNames()); - /// std::optional lets libpqxx to know if value is NULL std::vector> row(num_cols); @@ -156,21 +151,16 @@ public: } } - stream_inserter->write_values(row); + inserter->stream.write_values(row); } } - void writeSuffix() override { - if (stream_inserter) - { - stream_inserter->complete(); - work->commit(); - } + if (inserter) + inserter->complete(); } - /// Cannot just use serializeAsText for array data type even though it converts perfectly /// any dimension number array into text format, because it incloses in '[]' and for postgres it must be '{}'. /// Check if array[...] syntax from PostgreSQL will be applicable. @@ -207,7 +197,6 @@ public: writeChar('}', ostr); } - /// Conversion is done via column casting because with writeText(Array..) got incorrect conversion /// of Date and DateTime data types and it added extra quotes for values inside array. static std::string clickhouseToPostgresArray(const Array & array_field, const DataTypePtr & data_type) @@ -223,7 +212,6 @@ public: return '{' + std::string(ostr.str().begin() + 1, ostr.str().end() - 1) + '}'; } - static MutableColumnPtr createNested(DataTypePtr nested) { bool is_nullable = false; @@ -275,21 +263,34 @@ public: return nested_column; } - private: + struct StreamTo + { + pqxx::work tx; + pqxx::stream_to stream; + + StreamTo(pqxx::connection & connection, pqxx::table_path table_path, Names columns) + : tx(connection) + , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_path), connection.quote_columns(columns))) {} + + void complete() + { + stream.complete(); + tx.commit(); + } + }; + StorageMetadataPtr metadata_snapshot; postgres::ConnectionHolderPtr connection_holder; - std::string remote_table_name; - - std::unique_ptr work; - std::unique_ptr stream_inserter; + const String remote_table_name, remote_table_schema; + std::unique_ptr inserter; }; BlockOutputStreamPtr StoragePostgreSQL::write( const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr /* context */) { - return std::make_shared(metadata_snapshot, pool->get(), remote_table_name); + return std::make_shared(metadata_snapshot, pool->get(), remote_table_name, remote_table_schema); } diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 4daf46360d3..0be7f213de1 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -184,6 +184,11 @@ def test_non_default_scema(started_cluster): result = node1.query('SELECT * FROM test_pg_table_schema_with_dots') assert(result == expected) + cursor.execute('INSERT INTO "test_schema"."test_table" SELECT i FROM generate_series(100, 199) as t(i)') + result = node1.query('SELECT * FROM {}'.format(table_function)) + expected = node1.query('SELECT number FROM numbers(200)') + assert(result == expected) + def test_concurrent_queries(started_cluster): conn = get_postgres_conn(True) From c02088483d6bc46b66a2c131c627d85470cd773b Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 22 May 2021 17:24:40 +0000 Subject: [PATCH 062/491] Update CmakeLists.txt --- contrib/libpqxx-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libpqxx-cmake/CMakeLists.txt b/contrib/libpqxx-cmake/CMakeLists.txt index 4edef7bdd82..ae35538ccf4 100644 --- a/contrib/libpqxx-cmake/CMakeLists.txt +++ b/contrib/libpqxx-cmake/CMakeLists.txt @@ -64,7 +64,7 @@ set (HDRS add_library(libpqxx ${SRCS} ${HDRS}) target_link_libraries(libpqxx PUBLIC ${LIBPQ_LIBRARY}) -target_include_directories (libpqxx PRIVATE "${LIBRARY_DIR}/include") +target_include_directories (libpqxx SYSTEM PRIVATE "${LIBRARY_DIR}/include") # crutch set(CM_CONFIG_H_IN "${LIBRARY_DIR}/include/pqxx/config.h.in") From 54a385d1b0d391ebc43be59557ef2a6fed7b046f Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 31 May 2021 18:00:04 +0300 Subject: [PATCH 063/491] More fixes --- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 12 +++++++----- src/Interpreters/JoinToSubqueryTransformVisitor.cpp | 7 +++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index 463ad9dc9f0..1919e9d920c 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -80,20 +80,22 @@ void ApplyWithSubqueryVisitor::visit(ASTTableExpression & table, const Data & da void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data) { + /// Special CTE case, where the right argument of IN is alias (ASTIdentifier) from WITH clause. + if (checkFunctionIsInOrGlobalInOperator(func)) { auto & ast = func.arguments->children.at(1); - if (const auto * identifier = ast->as()) + if (const auto * identifier = ast->as()) { - auto table_id = identifier->getTableId(); - if (table_id.database_name.empty()) + if (identifier->isShort()) { - auto subquery_it = data.subqueries.find(table_id.table_name); + auto name = identifier->shortName(); + auto subquery_it = data.subqueries.find(name); if (subquery_it != data.subqueries.end()) { auto old_alias = func.arguments->children[1]->tryGetAlias(); func.arguments->children[1] = subquery_it->second->clone(); - func.arguments->children[1]->as().cte_name = table_id.table_name; + func.arguments->children[1]->as().cte_name = name; if (!old_alias.empty()) func.arguments->children[1]->setAlias(old_alias); } diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index ae7cbabee0a..ef69df16c8a 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -498,10 +498,9 @@ std::vector normalizeColumnNamesExtractNeeded( if (got_alias) { auto alias = aliases.find(ident->name())->second; - bool alias_equals_column_name = alias->ptr()->getColumnNameWithoutAlias() == ident->getColumnNameWithoutAlias(); - // FIXME: check test 01600_multiple_left_joins_with_aliases - // || (alias_table == IdentifierSemantic::getTableName(ident->ptr()) - // && ident->shortName() == alias->as()->shortName())) + auto alias_ident = alias->clone(); + alias_ident->as()->restoreTable(); + bool alias_equals_column_name = alias_ident->getColumnNameWithoutAlias() == ident->getColumnNameWithoutAlias(); if (!alias_equals_column_name) throw Exception("Alias clashes with qualified column '" + ident->name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); } From d0ad6d9cffbfa01524fcdb7047fa19933a7b1edb Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 31 May 2021 21:50:07 +0300 Subject: [PATCH 064/491] Fix all remaining tests --- src/Storages/VirtualColumnUtils.cpp | 2 +- tests/queries/0_stateless/01651_bugs_from_15889.sql | 11 ++++++----- .../0_stateless/01763_max_distributed_depth.sql | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 248b734195a..05023ee2c32 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -86,7 +86,7 @@ void buildSets(const ASTPtr & expression, ExpressionAnalyzer & analyzer) { const IAST & args = *func->arguments; const ASTPtr & arg = args.children.at(1); - if (arg->as() || arg->as()) + if (arg->as() || arg->as()) { analyzer.tryMakeSetForIndexFromSubquery(arg); } diff --git a/tests/queries/0_stateless/01651_bugs_from_15889.sql b/tests/queries/0_stateless/01651_bugs_from_15889.sql index 1fbf669a1b8..2f4c5a8261b 100644 --- a/tests/queries/0_stateless/01651_bugs_from_15889.sql +++ b/tests/queries/0_stateless/01651_bugs_from_15889.sql @@ -8,7 +8,8 @@ INSERT INTO xp SELECT '2020-01-01', number, '' FROM numbers(100000); CREATE TABLE xp_d AS xp ENGINE = Distributed(test_shard_localhost, currentDatabase(), xp); -SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- { serverError 20 } +-- FIXME: this query spontaneously returns either 8 or 20 error code. Looks like it's potentially flaky. +SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- { serverError 8 } SELECT count() FROM xp_d WHERE A GLOBAL IN (SELECT NULL); -- { serverError 53 } @@ -45,7 +46,7 @@ SYSTEM FLUSH LOGS; WITH concat(addressToLine(arrayJoin(trace) AS addr), '#') AS symbol SELECT count() > 7 FROM trace_log AS t -WHERE (query_id = +WHERE (query_id = ( SELECT [NULL, NULL, NULL, NULL, 0.00009999999747378752, NULL, NULL, NULL, NULL, NULL], @@ -60,7 +61,7 @@ WHERE (query_id = WITH addressToSymbol(arrayJoin(trace)) AS symbol SELECT count() > 0 FROM trace_log AS t -WHERE greaterOrEquals(event_date, ignore(ignore(ignore(NULL, '')), 256), yesterday()) AND (trace_type = 'Memory') AND (query_id = +WHERE greaterOrEquals(event_date, ignore(ignore(ignore(NULL, '')), 256), yesterday()) AND (trace_type = 'Memory') AND (query_id = ( SELECT ignore(ignore(ignore(ignore(65536)), ignore(65537), ignore(2)), ''), @@ -82,7 +83,7 @@ WITH ( WHERE current_database = currentDatabase() ORDER BY query_start_time DESC LIMIT 1 - ) AS time_with_microseconds, + ) AS time_with_microseconds, ( SELECT inf, @@ -101,7 +102,7 @@ WITH ( WHERE current_database = currentDatabase() ORDER BY query_start_time DESC LIMIT 1 - ) AS time_with_microseconds, + ) AS time_with_microseconds, ( SELECT query_start_time FROM system.query_log diff --git a/tests/queries/0_stateless/01763_max_distributed_depth.sql b/tests/queries/0_stateless/01763_max_distributed_depth.sql index 0bcb3cfe0b7..d1bb9e4be90 100644 --- a/tests/queries/0_stateless/01763_max_distributed_depth.sql +++ b/tests/queries/0_stateless/01763_max_distributed_depth.sql @@ -9,7 +9,7 @@ CREATE TABLE tt6 `status` String ) -ENGINE = Distributed('test_shard_localhost', currentDatabase(), 'tt6', rand()); +ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError 581 } From 365e52817bfbb0b2cccdf691b6b3f4eb2248524b Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 1 Jun 2021 14:20:03 +0300 Subject: [PATCH 065/491] More fixes due to "in" function arguments being incorrectly checked as ASTIdentifier --- src/Storages/MergeTree/KeyCondition.cpp | 2 +- src/Storages/transformQueryForExternalDatabase.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 268c45c305f..d2bf62b4e3b 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -848,7 +848,7 @@ bool KeyCondition::tryPrepareSetIndex( const ASTPtr & right_arg = args[1]; SetPtr prepared_set; - if (right_arg->as() || right_arg->as()) + if (right_arg->as() || right_arg->as()) { auto set_it = prepared_sets.find(PreparedSetKey::forSubquery(*right_arg)); if (set_it == prepared_sets.end()) diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index b3fe788d874..7a07cd94662 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -138,10 +138,9 @@ bool isCompatible(const IAST & node) if (name == "tuple" && function->arguments->children.size() <= 1) return false; - /// If the right hand side of IN is an identifier (example: x IN table), then it's not compatible. + /// If the right hand side of IN is a table identifier (example: x IN table), then it's not compatible. if ((name == "in" || name == "notIn") - && (function->arguments->children.size() != 2 - || function->arguments->children[1]->as())) + && (function->arguments->children.size() != 2 || function->arguments->children[1]->as())) return false; for (const auto & expr : function->arguments->children) From a68ba4c1d79f5b4e2af0c230bb43610b54546be8 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:16:20 +0300 Subject: [PATCH 066/491] Update 01651_bugs_from_15889.sql --- tests/queries/0_stateless/01651_bugs_from_15889.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01651_bugs_from_15889.sql b/tests/queries/0_stateless/01651_bugs_from_15889.sql index 2f4c5a8261b..d0f1006da95 100644 --- a/tests/queries/0_stateless/01651_bugs_from_15889.sql +++ b/tests/queries/0_stateless/01651_bugs_from_15889.sql @@ -9,7 +9,7 @@ INSERT INTO xp SELECT '2020-01-01', number, '' FROM numbers(100000); CREATE TABLE xp_d AS xp ENGINE = Distributed(test_shard_localhost, currentDatabase(), xp); -- FIXME: this query spontaneously returns either 8 or 20 error code. Looks like it's potentially flaky. -SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- { serverError 8 } +-- SELECT count(7 = (SELECT number FROM numbers(0) ORDER BY number ASC NULLS FIRST LIMIT 7)) FROM xp_d PREWHERE toYYYYMM(A) GLOBAL IN (SELECT NULL = (SELECT number FROM numbers(1) ORDER BY number DESC NULLS LAST LIMIT 1), toYYYYMM(min(A)) FROM xp_d) WHERE B > NULL; -- { serverError 8 } SELECT count() FROM xp_d WHERE A GLOBAL IN (SELECT NULL); -- { serverError 53 } From 2bd12a7612e3657632280526842afb1b612924af Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 2 Jun 2021 18:03:25 +0300 Subject: [PATCH 067/491] Fix DiskS3 read error due to broken getPosition() method in ReadBufferFromS3.cpp --- src/Disks/ReadIndirectBufferFromRemoteFS.cpp | 28 ++++++++++++------- src/Disks/ReadIndirectBufferFromRemoteFS.h | 4 ++- src/Disks/S3/DiskS3.cpp | 2 ++ src/IO/ReadBufferFromFileDecorator.cpp | 6 ++-- src/IO/ReadBufferFromFileDecorator.h | 4 +-- src/IO/ReadBufferFromS3.cpp | 29 ++++++++++---------- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 6d4764e4392..3b3efe78941 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -87,15 +87,14 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() { /// Find first available buffer that fits to given offset. if (!current_buf) + { current_buf = initialize(); + pos = working_buffer.begin(); + } /// If current buffer has remaining data - use it. - if (current_buf && current_buf->next()) - { - working_buffer = current_buf->buffer(); - absolute_position += working_buffer.size(); - return true; - } + if (current_buf) + return nextAndShiftPosition(); /// If there is no available buffers - nothing to read. if (current_buf_idx + 1 >= metadata.remote_fs_objects.size()) @@ -105,12 +104,21 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() const auto & path = metadata.remote_fs_objects[current_buf_idx].first; current_buf = createReadBuffer(path); - current_buf->next(); - working_buffer = current_buf->buffer(); - absolute_position += working_buffer.size(); + return nextAndShiftPosition(); +} - return true; +template +bool ReadIndirectBufferFromRemoteFS::nextAndShiftPosition() +{ + swap(*current_buf); + auto result = current_buf->next(); + swap(*current_buf); + + if (result) + absolute_position += working_buffer.size(); + + return result; } diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.h b/src/Disks/ReadIndirectBufferFromRemoteFS.h index f80406b5354..3ce7de295a8 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.h +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.h @@ -16,7 +16,7 @@ template class ReadIndirectBufferFromRemoteFS : public ReadBufferFromFileBase { public: - ReadIndirectBufferFromRemoteFS(IDiskRemote::Metadata metadata_); + explicit ReadIndirectBufferFromRemoteFS(IDiskRemote::Metadata metadata_); off_t seek(off_t offset_, int whence) override; @@ -32,6 +32,8 @@ protected: private: std::unique_ptr initialize(); + bool nextAndShiftPosition(); + bool nextImpl() override; size_t absolute_position = 0; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 89c2d20db9f..3ca8e07bba6 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -320,6 +320,8 @@ void DiskS3::startup() { auto settings = current_settings.get(); + settings->client->EnableRequestProcessing(); + if (!settings->send_metadata) return; diff --git a/src/IO/ReadBufferFromFileDecorator.cpp b/src/IO/ReadBufferFromFileDecorator.cpp index db0c5117d63..53fde7789bc 100644 --- a/src/IO/ReadBufferFromFileDecorator.cpp +++ b/src/IO/ReadBufferFromFileDecorator.cpp @@ -4,7 +4,7 @@ namespace DB { -ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptr impl_) +ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptr impl_) : impl(std::move(impl_)) { swap(*impl); @@ -13,7 +13,9 @@ ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptrgetFileName(); + if (ReadBufferFromFileBase * buffer = dynamic_cast(impl.get())) + return buffer->getFileName(); + return std::string(); } diff --git a/src/IO/ReadBufferFromFileDecorator.h b/src/IO/ReadBufferFromFileDecorator.h index 8b2796609b6..1122e02bb20 100644 --- a/src/IO/ReadBufferFromFileDecorator.h +++ b/src/IO/ReadBufferFromFileDecorator.h @@ -10,7 +10,7 @@ namespace DB class ReadBufferFromFileDecorator : public ReadBufferFromFileBase { public: - explicit ReadBufferFromFileDecorator(std::unique_ptr impl_); + explicit ReadBufferFromFileDecorator(std::unique_ptr impl_); std::string getFileName() const override; @@ -21,7 +21,7 @@ public: bool nextImpl() override; protected: - std::unique_ptr impl; + std::unique_ptr impl; }; } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 1e27b0284b7..86cf44178eb 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -43,12 +43,6 @@ ReadBufferFromS3::ReadBufferFromS3( bool ReadBufferFromS3::nextImpl() { - /// Restoring valid value of `count()` during `nextImpl()`. See `ReadBuffer::next()`. - pos = working_buffer.begin(); - - if (!impl) - impl = initialize(); - Stopwatch watch; bool next_result = false; @@ -84,25 +78,31 @@ bool ReadBufferFromS3::nextImpl() ProfileEvents::increment(ProfileEvents::S3ReadMicroseconds, watch.elapsedMicroseconds()); if (!next_result) return false; - internal_buffer = impl->buffer(); + + working_buffer = internal_buffer = impl->buffer(); + pos = working_buffer.begin(); ProfileEvents::increment(ProfileEvents::S3ReadBytes, internal_buffer.size()); - working_buffer = internal_buffer; + offset += working_buffer.size(); + return true; } off_t ReadBufferFromS3::seek(off_t offset_, int whence) { - if (impl) - throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - if (whence != SEEK_SET) throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); if (offset_ < 0) throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + if (impl) + { + impl.reset(); + pos = working_buffer.end(); + } + offset = offset_; return offset; @@ -110,18 +110,17 @@ off_t ReadBufferFromS3::seek(off_t offset_, int whence) off_t ReadBufferFromS3::getPosition() { - return offset + count(); + return offset - available(); } std::unique_ptr ReadBufferFromS3::initialize() { - LOG_TRACE(log, "Read S3 object. Bucket: {}, Key: {}, Offset: {}", bucket, key, getPosition()); + LOG_TRACE(log, "Read S3 object. Bucket: {}, Key: {}, Offset: {}", bucket, key, offset); Aws::S3::Model::GetObjectRequest req; req.SetBucket(bucket); req.SetKey(key); - if (getPosition()) - req.SetRange("bytes=" + std::to_string(getPosition()) + "-"); + req.SetRange(fmt::format("bytes={}-", offset)); Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req); From 977fbbf1e4b29c56cb4a016340c5e3e203466370 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 2 Jun 2021 20:17:26 +0300 Subject: [PATCH 068/491] Fix unit tests --- src/Parsers/MySQL/ASTCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/MySQL/ASTCreateQuery.cpp b/src/Parsers/MySQL/ASTCreateQuery.cpp index f45966c473e..22e13dbca54 100644 --- a/src/Parsers/MySQL/ASTCreateQuery.cpp +++ b/src/Parsers/MySQL/ASTCreateQuery.cpp @@ -63,7 +63,7 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (ParserKeyword("IF NOT EXISTS").ignore(pos, expected)) if_not_exists = true; - if (!ParserCompoundIdentifier(false).parse(pos, table, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) return false; if (ParserKeyword("LIKE").ignore(pos, expected)) From 6628b670df8edce5ec6849ea75b613a97c2d222e Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 3 Jun 2021 14:34:23 +0300 Subject: [PATCH 069/491] Fix reading from several buffers in ReadIndirectBufferFromRemoteFS --- src/Disks/ReadIndirectBufferFromRemoteFS.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 3b3efe78941..2bbc2e2197e 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -94,7 +94,11 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() /// If current buffer has remaining data - use it. if (current_buf) - return nextAndShiftPosition(); + { + bool result = nextAndShiftPosition(); + if (result) + return true; + } /// If there is no available buffers - nothing to read. if (current_buf_idx + 1 >= metadata.remote_fs_objects.size()) From bf5190cd3423d24c667324c707c8555887154657 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 3 Jun 2021 14:34:47 +0300 Subject: [PATCH 070/491] Fix reading from ReadBufferFromHDFS --- src/Storages/HDFS/ReadBufferFromHDFS.cpp | 37 +++++++++++++++--------- src/Storages/HDFS/ReadBufferFromHDFS.h | 2 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index f81d7736db3..d241bd07294 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes ReadBufferFromHDFS::~ReadBufferFromHDFS() = default; -struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl +struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory { /// HDFS create/open functions are not thread safe static std::mutex hdfs_init_mutex; @@ -37,8 +37,10 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl explicit ReadBufferFromHDFSImpl( const std::string & hdfs_uri_, const std::string & hdfs_file_path_, - const Poco::Util::AbstractConfiguration & config_) - : hdfs_uri(hdfs_uri_) + const Poco::Util::AbstractConfiguration & config_, + size_t buf_size_) + : BufferWithOwnMemory(buf_size_) + , hdfs_uri(hdfs_uri_) , hdfs_file_path(hdfs_file_path_) , builder(createHDFSBuilder(hdfs_uri_, config_)) { @@ -53,7 +55,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl hdfs_uri + hdfs_file_path, std::string(hdfsGetLastError())); } - ~ReadBufferFromHDFSImpl() + ~ReadBufferFromHDFSImpl() override { std::lock_guard lock(hdfs_init_mutex); hdfsCloseFile(fs.get(), fin); @@ -69,7 +71,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Fail to seek HDFS file: {}, error: {}", hdfs_uri, std::string(hdfsGetLastError())); } - int read(char * start, size_t size) + bool nextImpl() override { if (!initialized) { @@ -77,15 +79,19 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl initialized = true; } - int bytes_read = hdfsRead(fs.get(), fin, start, size); + int bytes_read = hdfsRead(fs.get(), fin, internal_buffer.begin(), internal_buffer.size()); if (bytes_read < 0) throw Exception(ErrorCodes::NETWORK_ERROR, "Fail to read from HDFS: {}, file path: {}. Error: {}", hdfs_uri, hdfs_file_path, std::string(hdfsGetLastError())); + + working_buffer.resize(bytes_read); + offset += bytes_read; + return bytes_read; } - int seek(off_t offset_, int whence) + off_t seek(off_t offset_, int whence) override { if (initialized) throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); @@ -101,7 +107,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl return offset; } - int tell() const + off_t getPosition() override { return offset; } @@ -115,18 +121,21 @@ ReadBufferFromHDFS::ReadBufferFromHDFS( const String & hdfs_file_path_, const Poco::Util::AbstractConfiguration & config_, size_t buf_size_) - : BufferWithOwnMemory(buf_size_) - , impl(std::make_unique(hdfs_uri_, hdfs_file_path_, config_)) + : SeekableReadBuffer(nullptr, 0) + , impl(std::make_unique(hdfs_uri_, hdfs_file_path_, config_, buf_size_)) { } bool ReadBufferFromHDFS::nextImpl() { - int bytes_read = impl->read(internal_buffer.begin(), internal_buffer.size()); + auto result = impl->next(); - if (bytes_read) - working_buffer.resize(bytes_read); + if (result) + { + working_buffer = internal_buffer = impl->buffer(); + pos = working_buffer.begin(); + } else return false; return true; @@ -141,7 +150,7 @@ off_t ReadBufferFromHDFS::seek(off_t off, int whence) off_t ReadBufferFromHDFS::getPosition() { - return impl->tell() + count(); + return impl->getPosition() - available(); } } diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.h b/src/Storages/HDFS/ReadBufferFromHDFS.h index 498056ea376..b06e0376fcd 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.h +++ b/src/Storages/HDFS/ReadBufferFromHDFS.h @@ -19,7 +19,7 @@ namespace DB /** Accepts HDFS path to file and opens it. * Closes file by himself (thus "owns" a file descriptor). */ -class ReadBufferFromHDFS : public BufferWithOwnMemory +class ReadBufferFromHDFS : public SeekableReadBuffer { struct ReadBufferFromHDFSImpl; From 25f3efde2be1164957b2e83c688d2d25c7f901b6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 30 May 2021 14:42:16 +0300 Subject: [PATCH 071/491] Add a test for Materialized(Distributed()) with JOIN and GROUP BY --- ...90_materialized_distributed_join.reference | 8 +++++++ .../01890_materialized_distributed_join.sql | 24 +++++++++++++++++++ .../queries/0_stateless/arcadia_skip_list.txt | 1 + 3 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/01890_materialized_distributed_join.reference create mode 100644 tests/queries/0_stateless/01890_materialized_distributed_join.sql diff --git a/tests/queries/0_stateless/01890_materialized_distributed_join.reference b/tests/queries/0_stateless/01890_materialized_distributed_join.reference new file mode 100644 index 00000000000..315ebca7e7a --- /dev/null +++ b/tests/queries/0_stateless/01890_materialized_distributed_join.reference @@ -0,0 +1,8 @@ +1 1 1 2 +1 1 1 2 +1 1 1 2 +1 1 1 2 +1 1 1 2 +1 1 1 2 +2 +4 diff --git a/tests/queries/0_stateless/01890_materialized_distributed_join.sql b/tests/queries/0_stateless/01890_materialized_distributed_join.sql new file mode 100644 index 00000000000..0020419981b --- /dev/null +++ b/tests/queries/0_stateless/01890_materialized_distributed_join.sql @@ -0,0 +1,24 @@ +drop table if exists test_distributed; +drop table if exists test_source; +drop table if exists test_shard; +drop table if exists test_local; + +create table test_shard (k UInt64, v UInt64) ENGINE Memory(); +create table test_local (k UInt64, v UInt64) ENGINE Memory(); +create table test_source (k UInt64, v UInt64) ENGINE Memory(); + +insert into test_shard values (1, 1); +insert into test_local values (1, 2); + +create materialized view test_distributed engine Distributed('test_cluster_two_shards', currentDatabase(), 'test_shard', k) as select k, v from test_source; + +select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v; +select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v order by td.v; +select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v order by tl.v; +select sum(td.v) from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v group by tl.k; +select sum(tl.v) from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v group by td.k; + +drop table test_distributed; +drop table test_source; +drop table test_shard; +drop table test_local; diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index a59c1a72a52..39cc75d2f5d 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -240,3 +240,4 @@ 01880_remote_ipv6 01882_scalar_subquery_exception 01882_check_max_parts_to_merge_at_once +01890_materialized_distributed_join From 578ecc1645dd488b1f9e5974f43e4a2153ea1a57 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 30 May 2021 17:43:54 +0300 Subject: [PATCH 072/491] removeJoin: remove joined columns --- .../getHeaderForProcessingStage.cpp | 63 ++++++++++++++++--- .../getHeaderForProcessingStage.h | 3 +- src/Storages/StorageMerge.cpp | 48 +------------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index 9c7c86a0b88..821f41c086c 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include #include @@ -23,16 +25,58 @@ bool hasJoin(const ASTSelectQuery & select) } /// Rewrite original query removing joined tables from it -bool removeJoin(ASTSelectQuery & select) +bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, ContextPtr context) { - if (hasJoin(select)) + if (!hasJoin(select)) + return false; + + select.tables()->children.resize(1); + + /// Also remove GROUP BY cause ExpressionAnalyzer would check if it has all aggregate columns but joined columns would be missed. + select.setExpression(ASTSelectQuery::Expression::GROUP_BY, {}); + rewriter_result.aggregates.clear(); + + /// Replace select list to remove joined columns + auto select_list = std::make_shared(); + for (const auto & column : rewriter_result.required_source_columns) + select_list->children.emplace_back(std::make_shared(column.name)); + + select.setExpression(ASTSelectQuery::Expression::SELECT, select_list); + + const DB::IdentifierMembershipCollector membership_collector{select, context}; + + /// Remove unknown identifiers from where, leave only ones from left table + auto replace_where = [&membership_collector](ASTSelectQuery & query, ASTSelectQuery::Expression expr) { - /// The most simple temporary solution: leave only the first table in query. - /// TODO: we also need to remove joined columns and related functions (taking in account aliases if any). - select.tables()->children.resize(1); - return true; - } - return false; + auto where = query.getExpression(expr, false); + if (!where) + return; + + const size_t left_table_pos = 0; + /// Test each argument of `and` function and select ones related to only left table + std::shared_ptr new_conj = makeASTFunction("and"); + for (const auto & node : collectConjunctions(where)) + { + if (membership_collector.getIdentsMembership(node) == left_table_pos) + new_conj->arguments->children.push_back(std::move(node)); + } + + if (new_conj->arguments->children.empty()) + /// No identifiers from left table + query.setExpression(expr, {}); + else if (new_conj->arguments->children.size() == 1) + /// Only one expression, lift from `and` + query.setExpression(expr, std::move(new_conj->arguments->children[0])); + else + /// Set new expression + query.setExpression(expr, std::move(new_conj)); + }; + replace_where(select, ASTSelectQuery::Expression::WHERE); + replace_where(select, ASTSelectQuery::Expression::PREWHERE); + select.setExpression(ASTSelectQuery::Expression::HAVING, {}); + select.setExpression(ASTSelectQuery::Expression::ORDER_BY, {}); + + return true; } Block getHeaderForProcessingStage( @@ -72,7 +116,8 @@ Block getHeaderForProcessingStage( case QueryProcessingStage::MAX: { auto query = query_info.query->clone(); - removeJoin(*query->as()); + TreeRewriterResult new_rewriter_result = *query_info.syntax_analyzer_result; + removeJoin(*query->as(), new_rewriter_result, context); auto stream = std::make_shared( metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID())); diff --git a/src/Interpreters/getHeaderForProcessingStage.h b/src/Interpreters/getHeaderForProcessingStage.h index 75a89bc5d39..54a1126a3df 100644 --- a/src/Interpreters/getHeaderForProcessingStage.h +++ b/src/Interpreters/getHeaderForProcessingStage.h @@ -13,10 +13,11 @@ class IStorage; struct StorageInMemoryMetadata; using StorageMetadataPtr = std::shared_ptr; struct SelectQueryInfo; +struct TreeRewriterResult; class ASTSelectQuery; bool hasJoin(const ASTSelectQuery & select); -bool removeJoin(ASTSelectQuery & select); +bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, ContextPtr context); Block getHeaderForProcessingStage( const IStorage & storage, diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 15d520c13aa..5a8864141c4 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -46,54 +46,8 @@ namespace TreeRewriterResult modifySelect(ASTSelectQuery & select, const TreeRewriterResult & rewriter_result, ContextPtr context) { - TreeRewriterResult new_rewriter_result = rewriter_result; - if (removeJoin(select)) - { - /// Also remove GROUP BY cause ExpressionAnalyzer would check if it has all aggregate columns but joined columns would be missed. - select.setExpression(ASTSelectQuery::Expression::GROUP_BY, {}); - new_rewriter_result.aggregates.clear(); - - /// Replace select list to remove joined columns - auto select_list = std::make_shared(); - for (const auto & column : rewriter_result.required_source_columns) - select_list->children.emplace_back(std::make_shared(column.name)); - - select.setExpression(ASTSelectQuery::Expression::SELECT, select_list); - - const DB::IdentifierMembershipCollector membership_collector{select, context}; - - /// Remove unknown identifiers from where, leave only ones from left table - auto replace_where = [&membership_collector](ASTSelectQuery & query, ASTSelectQuery::Expression expr) - { - auto where = query.getExpression(expr, false); - if (!where) - return; - - const size_t left_table_pos = 0; - /// Test each argument of `and` function and select ones related to only left table - std::shared_ptr new_conj = makeASTFunction("and"); - for (const auto & node : collectConjunctions(where)) - { - if (membership_collector.getIdentsMembership(node) == left_table_pos) - new_conj->arguments->children.push_back(std::move(node)); - } - - if (new_conj->arguments->children.empty()) - /// No identifiers from left table - query.setExpression(expr, {}); - else if (new_conj->arguments->children.size() == 1) - /// Only one expression, lift from `and` - query.setExpression(expr, std::move(new_conj->arguments->children[0])); - else - /// Set new expression - query.setExpression(expr, std::move(new_conj)); - }; - replace_where(select,ASTSelectQuery::Expression::WHERE); - replace_where(select,ASTSelectQuery::Expression::PREWHERE); - select.setExpression(ASTSelectQuery::Expression::HAVING, {}); - select.setExpression(ASTSelectQuery::Expression::ORDER_BY, {}); - } + removeJoin(select, new_rewriter_result, context); return new_rewriter_result; } From 01e6a4d919e106ced52f011ac6482848ad9ff61e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 2 Jun 2021 09:28:12 +0300 Subject: [PATCH 073/491] Do not try convert columns that does not exists in materialized view --- src/Storages/StorageMaterializedView.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index a66b24ff931..d11d0a2ae1a 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -166,6 +166,16 @@ void StorageMaterializedView::read( { auto mv_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, local_context, processed_stage); auto target_header = query_plan.getCurrentDataStream().header; + + /// No need to convert columns that does not exists in MV + std::set target_only_positions; + for (const auto & column : target_header) + { + if (!mv_header.has(column.name)) + target_only_positions.insert(target_header.getPositionByName(column.name)); + } + target_header.erase(target_only_positions); + if (!blocksHaveEqualStructure(mv_header, target_header)) { auto converting_actions = ActionsDAG::makeConvertingActions(target_header.getColumnsWithTypeAndName(), From 6cb2e81d48a574f29ae3e0b3a015916ba20aa798 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 2 Jun 2021 10:31:28 +0300 Subject: [PATCH 074/491] Do not try convert columns that does not exists in the result block --- src/Storages/StorageMaterializedView.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d11d0a2ae1a..733cbaed8bb 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -176,6 +176,19 @@ void StorageMaterializedView::read( } target_header.erase(target_only_positions); + /// No need to convert columns that does not exists in the result header. + /// + /// Distributed storage may process query up to the specific stage, and + /// so the result header may not include all the columns from the + /// materialized view. + std::set source_only_positions; + for (const auto & column : mv_header) + { + if (!target_header.has(column.name)) + source_only_positions.insert(mv_header.getPositionByName(column.name)); + } + mv_header.erase(source_only_positions); + if (!blocksHaveEqualStructure(mv_header, target_header)) { auto converting_actions = ActionsDAG::makeConvertingActions(target_header.getColumnsWithTypeAndName(), From ea72e669962f0fc13ab6645f0b138ae216616d62 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 3 Jun 2021 21:07:19 +0300 Subject: [PATCH 075/491] Convert 01890_materialized_distributed_join to .sh for explicit database for JOIN --- .../01890_materialized_distributed_join.sh | 34 +++++++++++++++++++ .../01890_materialized_distributed_join.sql | 24 ------------- 2 files changed, 34 insertions(+), 24 deletions(-) create mode 100755 tests/queries/0_stateless/01890_materialized_distributed_join.sh delete mode 100644 tests/queries/0_stateless/01890_materialized_distributed_join.sql diff --git a/tests/queries/0_stateless/01890_materialized_distributed_join.sh b/tests/queries/0_stateless/01890_materialized_distributed_join.sh new file mode 100755 index 00000000000..e9808a80db1 --- /dev/null +++ b/tests/queries/0_stateless/01890_materialized_distributed_join.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# FIXME: this is an .sh test because JOIN with Distributed in the left will use default database for the right. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " + drop table if exists test_distributed; + drop table if exists test_source; + drop table if exists test_shard; + drop table if exists test_local; + + create table test_shard (k UInt64, v UInt64) ENGINE Memory(); + create table test_local (k UInt64, v UInt64) ENGINE Memory(); + create table test_source (k UInt64, v UInt64) ENGINE Memory(); + + insert into test_shard values (1, 1); + insert into test_local values (1, 2); + + create materialized view test_distributed engine Distributed('test_cluster_two_shards', $CLICKHOUSE_DATABASE, 'test_shard', k) as select k, v from test_source; + + select * from test_distributed td asof join $CLICKHOUSE_DATABASE.test_local tl on td.k = tl.k and td.v < tl.v; + select * from test_distributed td asof join $CLICKHOUSE_DATABASE.test_local tl on td.k = tl.k and td.v < tl.v order by td.v; + select * from test_distributed td asof join $CLICKHOUSE_DATABASE.test_local tl on td.k = tl.k and td.v < tl.v order by tl.v; + select sum(td.v) from test_distributed td asof join $CLICKHOUSE_DATABASE.test_local tl on td.k = tl.k and td.v < tl.v group by tl.k; + select sum(tl.v) from test_distributed td asof join $CLICKHOUSE_DATABASE.test_local tl on td.k = tl.k and td.v < tl.v group by td.k; + + drop table test_distributed; + drop table test_source; + drop table test_shard; + drop table test_local; +" diff --git a/tests/queries/0_stateless/01890_materialized_distributed_join.sql b/tests/queries/0_stateless/01890_materialized_distributed_join.sql deleted file mode 100644 index 0020419981b..00000000000 --- a/tests/queries/0_stateless/01890_materialized_distributed_join.sql +++ /dev/null @@ -1,24 +0,0 @@ -drop table if exists test_distributed; -drop table if exists test_source; -drop table if exists test_shard; -drop table if exists test_local; - -create table test_shard (k UInt64, v UInt64) ENGINE Memory(); -create table test_local (k UInt64, v UInt64) ENGINE Memory(); -create table test_source (k UInt64, v UInt64) ENGINE Memory(); - -insert into test_shard values (1, 1); -insert into test_local values (1, 2); - -create materialized view test_distributed engine Distributed('test_cluster_two_shards', currentDatabase(), 'test_shard', k) as select k, v from test_source; - -select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v; -select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v order by td.v; -select * from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v order by tl.v; -select sum(td.v) from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v group by tl.k; -select sum(tl.v) from test_distributed td asof join test_local tl on td.k = tl.k and td.v < tl.v group by td.k; - -drop table test_distributed; -drop table test_source; -drop table test_shard; -drop table test_local; From 4d91dfda7eb9319adc0d08ae7bd986b423e8bf43 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Fri, 4 Jun 2021 17:03:11 +0100 Subject: [PATCH 076/491] Replace if over an enum with a switch to make it hard to miss new cases --- src/Interpreters/InterpreterAlterQuery.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 25cb679094b..33697db95c0 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -289,15 +289,20 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS } case ASTAlterCommand::MOVE_PARTITION: { - if ((command.move_destination_type == DataDestinationType::DISK) - || (command.move_destination_type == DataDestinationType::VOLUME)) + switch (command.move_destination_type) { - required_access.emplace_back(AccessType::ALTER_MOVE_PARTITION, database, table); - } - else if (command.move_destination_type == DataDestinationType::TABLE) - { - required_access.emplace_back(AccessType::SELECT | AccessType::ALTER_DELETE, database, table); - required_access.emplace_back(AccessType::INSERT, command.to_database, command.to_table); + case DataDestinationType::DISK: [[fallthrough]]; + case DataDestinationType::VOLUME: + required_access.emplace_back(AccessType::ALTER_MOVE_PARTITION, database, table); + break; + case DataDestinationType::TABLE: + required_access.emplace_back(AccessType::SELECT | AccessType::ALTER_DELETE, database, table); + required_access.emplace_back(AccessType::INSERT, command.to_database, command.to_table); + break; + case DataDestinationType::SHARD: + break; + case DataDestinationType::DELETE: + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected destination type for command."); } break; } From af311c864281f686670e69c0bc01da6c4949a935 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Fri, 4 Jun 2021 17:25:40 +0100 Subject: [PATCH 077/491] Add acl for move partition between shards We can not easily verify permissions on destination shard and instead we require a custom grant that can be given to superadmins. --- src/Access/AccessType.h | 3 +++ src/Interpreters/InterpreterAlterQuery.cpp | 2 ++ tests/queries/0_stateless/01271_show_privileges.reference | 1 + 3 files changed, 6 insertions(+) diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index 22d99112cb7..cef2de12b30 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -102,6 +102,9 @@ enum class AccessType M(KILL_QUERY, "", GLOBAL, ALL) /* allows to kill a query started by another user (anyone can kill his own queries) */\ \ + M(MOVE_PARTITION_BETWEEN_SHARDS, "", GLOBAL, ALL) /* required to be able to move a part/partition to a table + identified by it's ZooKeeper path */\ + \ M(CREATE_USER, "", GLOBAL, ACCESS_MANAGEMENT) \ M(ALTER_USER, "", GLOBAL, ACCESS_MANAGEMENT) \ M(DROP_USER, "", GLOBAL, ACCESS_MANAGEMENT) \ diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 33697db95c0..3871d9f3295 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -300,6 +300,8 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS required_access.emplace_back(AccessType::INSERT, command.to_database, command.to_table); break; case DataDestinationType::SHARD: + required_access.emplace_back(AccessType::SELECT | AccessType::ALTER_DELETE, database, table); + required_access.emplace_back(AccessType::MOVE_PARTITION_BETWEEN_SHARDS); break; case DataDestinationType::DELETE: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected destination type for command."); diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 59b37369739..0ab0d57ebcf 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -54,6 +54,7 @@ DROP [] \N ALL TRUNCATE ['TRUNCATE TABLE'] TABLE ALL OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL KILL QUERY [] GLOBAL ALL +MOVE PARTITION BETWEEN SHARDS [] GLOBAL ALL CREATE USER [] GLOBAL ACCESS MANAGEMENT ALTER USER [] GLOBAL ACCESS MANAGEMENT DROP USER [] GLOBAL ACCESS MANAGEMENT From c015ec7be9c4c86d4951a20652ab771246ec3310 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 5 Jun 2021 14:20:39 +0300 Subject: [PATCH 078/491] Fix serialization of splitted nested messages in Protobuf format. --- src/Formats/ProtobufSerializer.cpp | 187 +++++++++++------- ..._protobuf_format_splitted_nested.reference | 56 ++++++ .../00825_protobuf_format_splitted_nested.sh | 73 +++++++ ...0825_protobuf_format_splitted_nested.proto | 72 +++++++ 4 files changed, 317 insertions(+), 71 deletions(-) create mode 100644 tests/queries/0_stateless/00825_protobuf_format_splitted_nested.reference create mode 100755 tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh create mode 100644 tests/queries/0_stateless/format_schemas/00825_protobuf_format_splitted_nested.proto diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 962df507f82..e9d0d55af9a 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -40,7 +40,7 @@ # include # include # include - +# include # include namespace DB @@ -2051,8 +2051,7 @@ namespace public: struct FieldDesc { - size_t column_index; - size_t num_columns; + std::vector column_indices; const FieldDescriptor * field_descriptor; std::unique_ptr field_serializer; }; @@ -2070,7 +2069,7 @@ namespace { field_infos.reserve(field_descs_.size()); for (auto & desc : field_descs_) - field_infos.emplace_back(desc.column_index, desc.num_columns, *desc.field_descriptor, std::move(desc.field_serializer)); + field_infos.emplace_back(std::move(desc.column_indices), *desc.field_descriptor, std::move(desc.field_serializer)); std::sort(field_infos.begin(), field_infos.end(), [](const FieldInfo & lhs, const FieldInfo & rhs) { return lhs.field_tag < rhs.field_tag; }); @@ -2083,22 +2082,28 @@ namespace { columns.assign(columns_, columns_ + num_columns_); + std::vector field_columns; for (const FieldInfo & info : field_infos) - info.field_serializer->setColumns(columns.data() + info.column_index, info.num_columns); + { + field_columns.clear(); + for (size_t column_index : info.column_indices) + { + field_columns.emplace_back(columns_[column_index]); + } + info.field_serializer->setColumns(field_columns.data(), field_columns.size()); + } if (reader) { - missing_column_indices.clear(); - missing_column_indices.reserve(num_columns_); - size_t current_idx = 0; + missing_column_indices.resize(num_columns_); + for (size_t column_index : ext::range(num_columns_)) + missing_column_indices[column_index] = column_index; for (const FieldInfo & info : field_infos) { - while (current_idx < info.column_index) - missing_column_indices.push_back(current_idx++); - current_idx = info.column_index + info.num_columns; + for (size_t column_index : info.column_indices) + missing_column_indices[column_index] = static_cast(-1); } - while (current_idx < num_columns_) - missing_column_indices.push_back(current_idx++); + boost::range::remove_erase(missing_column_indices, static_cast(-1)); } } @@ -2231,20 +2236,17 @@ namespace struct FieldInfo { FieldInfo( - size_t column_index_, - size_t num_columns_, + std::vector column_indices_, const FieldDescriptor & field_descriptor_, std::unique_ptr field_serializer_) - : column_index(column_index_) - , num_columns(num_columns_) + : column_indices(std::move(column_indices_)) , field_descriptor(&field_descriptor_) , field_tag(field_descriptor_.number()) , should_pack_repeated(shouldPackRepeated(field_descriptor_)) , field_serializer(std::move(field_serializer_)) { } - size_t column_index; - size_t num_columns; + std::vector column_indices; const FieldDescriptor * field_descriptor; int field_tag; bool should_pack_repeated; @@ -2581,6 +2583,38 @@ namespace return !out_field_descriptors_with_suffixes.empty(); } + /// Removes TypeIndex::Array from the specified vector of data types, + /// and also removes corresponding elements from two other vectors. + template + static void removeNonArrayElements(DataTypes & data_types, std::vector & elements1, std::vector & elements2) + { + size_t initial_size = data_types.size(); + assert(initial_size == elements1.size() && initial_size == elements2.size()); + data_types.reserve(initial_size * 2); + elements1.reserve(initial_size * 2); + elements2.reserve(initial_size * 2); + for (size_t i : ext::range(initial_size)) + { + if (data_types[i]->getTypeId() == TypeIndex::Array) + { + data_types.push_back(std::move(data_types[i])); + elements1.push_back(std::move(elements1[i])); + elements2.push_back(std::move(elements2[i])); + } + } + data_types.erase(data_types.begin(), data_types.begin() + initial_size); + elements1.erase(elements1.begin(), elements1.begin() + initial_size); + elements2.erase(elements2.begin(), elements2.begin() + initial_size); + } + + /// Treats specified column indices as indices in another vector of column indices. + /// Useful for handling of nested messages. + static void transformColumnIndices(std::vector & column_indices, const std::vector & outer_indices) + { + for (size_t & idx : column_indices) + idx = outer_indices[idx]; + } + /// Builds a serializer for a protobuf message (root or nested). template std::unique_ptr buildMessageSerializerImpl( @@ -2598,9 +2632,8 @@ namespace used_column_indices.clear(); used_column_indices.reserve(num_columns); - auto add_field_serializer = [&](size_t column_index_, - const std::string_view & column_name_, - size_t num_columns_, + auto add_field_serializer = [&](const std::string_view & column_name_, + std::vector column_indices_, const FieldDescriptor & field_descriptor_, std::unique_ptr field_serializer_) { @@ -2608,24 +2641,30 @@ namespace if (it != field_descriptors_in_use.end()) { throw Exception( - "Multiple columns (" + backQuote(StringRef{field_descriptors_in_use[&field_descriptor_]}) + ", " + "Multiple columns (" + backQuote(StringRef{it->second}) + ", " + backQuote(StringRef{column_name_}) + ") cannot be serialized to a single protobuf field " + quoteString(field_descriptor_.full_name()), ErrorCodes::MULTIPLE_COLUMNS_SERIALIZED_TO_SAME_PROTOBUF_FIELD); } - field_descs.push_back({column_index_, num_columns_, &field_descriptor_, std::move(field_serializer_)}); + for (size_t column_index : column_indices_) + { + /// Keep `used_column_indices` sorted. + used_column_indices.insert(boost::range::upper_bound(used_column_indices, column_index), column_index); + } + field_descs.push_back({std::move(column_indices_), &field_descriptor_, std::move(field_serializer_)}); field_descriptors_in_use.emplace(&field_descriptor_, column_name_); }; std::vector> field_descriptors_with_suffixes; /// We're going through all the passed columns. - size_t column_idx = 0; - size_t next_column_idx = 1; - for (; column_idx != num_columns; column_idx = next_column_idx++) + for (size_t column_idx : ext::range(num_columns)) { - auto column_name = column_names[column_idx]; + if (boost::range::binary_search(used_column_indices, column_idx)) + continue; + + const auto & column_name = column_names[column_idx]; const auto & data_type = data_types[column_idx]; if (!findFieldsByColumnName(column_name, message_descriptor, field_descriptors_with_suffixes)) @@ -2639,8 +2678,7 @@ namespace if (field_serializer) { - add_field_serializer(column_idx, column_name, 1, field_descriptor, std::move(field_serializer)); - used_column_indices.push_back(column_idx); + add_field_serializer(column_name, {column_idx}, field_descriptor, std::move(field_serializer)); continue; } } @@ -2650,42 +2688,53 @@ namespace if (!suffix.empty()) { /// Complex case: one or more columns are serialized as a nested message. - std::vector names_relative_to_nested_message; - names_relative_to_nested_message.reserve(num_columns - column_idx); - names_relative_to_nested_message.emplace_back(suffix); + std::vector nested_column_indices; + std::vector nested_column_names; + nested_column_indices.reserve(num_columns - used_column_indices.size()); + nested_column_names.reserve(num_columns - used_column_indices.size()); + nested_column_indices.push_back(column_idx); + nested_column_names.push_back(suffix); for (size_t j : ext::range(column_idx + 1, num_columns)) { - std::string_view next_suffix; - if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, next_suffix)) - break; - names_relative_to_nested_message.emplace_back(next_suffix); + if (boost::range::binary_search(used_column_indices, j)) + continue; + std::string_view other_suffix; + if (!columnNameStartsWithFieldName(column_names[j], *field_descriptor, other_suffix)) + continue; + nested_column_indices.push_back(j); + nested_column_names.push_back(other_suffix); } - /// Now we have up to `names_relative_to_nested_message.size()` sequential columns + DataTypes nested_data_types; + nested_data_types.reserve(nested_column_indices.size()); + for (size_t j : nested_column_indices) + nested_data_types.push_back(data_types[j]); + + /// Now we have up to `nested_message_column_names.size()` columns /// which can be serialized as a nested message. - /// Calculate how many of those sequential columns are arrays. - size_t num_arrays = 0; - for (size_t j : ext::range(column_idx, column_idx + names_relative_to_nested_message.size())) + /// We will try to serialize those columns as one nested message, + /// then, if failed, as an array of nested messages (on condition if those columns are array). + bool has_fallback_to_array_of_nested_messages = false; + if (field_descriptor->is_repeated()) { - if (data_types[j]->getTypeId() != TypeIndex::Array) - break; - ++num_arrays; + bool has_arrays + = boost::range::find_if( + nested_data_types, [](const DataTypePtr & dt) { return (dt->getTypeId() == TypeIndex::Array); }) + != nested_data_types.end(); + if (has_arrays) + has_fallback_to_array_of_nested_messages = true; } - /// We will try to serialize the sequential columns as one nested message, - /// then, if failed, as an array of nested messages (on condition those columns are array). - bool has_fallback_to_array_of_nested_messages = num_arrays && field_descriptor->is_repeated(); - - /// Try to serialize the sequential columns as one nested message. + /// Try to serialize those columns as one nested message. try { std::vector used_column_indices_in_nested; auto nested_message_serializer = buildMessageSerializerImpl( - names_relative_to_nested_message.size(), - names_relative_to_nested_message.data(), - &data_types[column_idx], + nested_column_names.size(), + nested_column_names.data(), + nested_data_types.data(), used_column_indices_in_nested, *field_descriptor->message_type(), false, @@ -2693,11 +2742,12 @@ namespace if (nested_message_serializer) { - for (size_t & idx_in_nested : used_column_indices_in_nested) - used_column_indices.push_back(idx_in_nested + column_idx); - - next_column_idx = used_column_indices.back() + 1; - add_field_serializer(column_idx, column_name, next_column_idx - column_idx, *field_descriptor, std::move(nested_message_serializer)); + transformColumnIndices(used_column_indices_in_nested, nested_column_indices); + add_field_serializer( + column_name, + std::move(used_column_indices_in_nested), + *field_descriptor, + std::move(nested_message_serializer)); break; } } @@ -2709,17 +2759,16 @@ namespace if (has_fallback_to_array_of_nested_messages) { - /// Try to serialize the sequential columns as an array of nested messages. - DataTypes array_nested_data_types; - array_nested_data_types.reserve(num_arrays); - for (size_t j : ext::range(column_idx, column_idx + num_arrays)) - array_nested_data_types.emplace_back(assert_cast(*data_types[j]).getNestedType()); + /// Try to serialize those columns as an array of nested messages. + removeNonArrayElements(nested_data_types, nested_column_names, nested_column_indices); + for (DataTypePtr & dt : nested_data_types) + dt = assert_cast(*dt).getNestedType(); std::vector used_column_indices_in_nested; auto nested_message_serializer = buildMessageSerializerImpl( - array_nested_data_types.size(), - names_relative_to_nested_message.data(), - array_nested_data_types.data(), + nested_column_names.size(), + nested_column_names.data(), + nested_data_types.data(), used_column_indices_in_nested, *field_descriptor->message_type(), false, @@ -2728,12 +2777,8 @@ namespace if (nested_message_serializer) { auto field_serializer = std::make_unique(std::move(nested_message_serializer)); - - for (size_t & idx_in_nested : used_column_indices_in_nested) - used_column_indices.push_back(idx_in_nested + column_idx); - - next_column_idx = used_column_indices.back() + 1; - add_field_serializer(column_idx, column_name, next_column_idx - column_idx, *field_descriptor, std::move(field_serializer)); + transformColumnIndices(used_column_indices_in_nested, nested_column_indices); + add_field_serializer(column_name, std::move(used_column_indices_in_nested), *field_descriptor, std::move(field_serializer)); break; } } diff --git a/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.reference b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.reference new file mode 100644 index 00000000000..2fa8a590122 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.reference @@ -0,0 +1,56 @@ +tags for first fixed value 1622559733 920 1 79034445678 250208889765444 35655678903421 79991232222 250 20 18122 22010 text for the first fixed value \N \N \N \N \N \N \N \N \N \N \N 3 172.18.20.11 47855 32705 26855 51940 0x1dbb09 _49597 msc_number_52317 0x750x830xa50xb 31453 49538 1 522d + +Binary representation: +00000000 f7 01 0a 1a 74 61 67 73 20 66 6f 72 20 66 69 72 |....tags for fir| +00000010 73 74 20 66 69 78 65 64 20 76 61 6c 75 65 10 f5 |st fixed value..| +00000020 97 d9 85 06 18 98 07 20 01 28 01 32 0b 37 39 30 |....... .(.2.790| +00000030 33 34 34 34 35 36 37 38 3a 0f 32 35 30 32 30 38 |34445678:.250208| +00000040 38 38 39 37 36 35 34 34 34 42 0e 33 35 36 35 35 |889765444B.35655| +00000050 36 37 38 39 30 33 34 32 31 4a 0b 37 39 39 39 31 |678903421J.79991| +00000060 32 33 32 32 32 32 50 fa 01 58 14 60 ca 8d 01 68 |232222P..X.`...h| +00000070 fa ab 01 72 1e 74 65 78 74 20 66 6f 72 20 74 68 |...r.text for th| +00000080 65 20 66 69 72 73 74 20 66 69 78 65 64 20 76 61 |e first fixed va| +00000090 6c 75 65 aa 06 63 08 03 1a 0c 31 37 32 2e 31 38 |lue..c....172.18| +000000a0 2e 32 30 2e 31 31 20 ef f5 02 28 c1 ff 01 30 e7 |.20.11 ...(...0.| +000000b0 d1 01 38 e4 95 03 42 08 30 78 31 64 62 62 30 39 |..8...B.0x1dbb09| +000000c0 4a 06 5f 34 39 35 39 37 52 10 6d 73 63 5f 6e 75 |J._49597R.msc_nu| +000000d0 6d 62 65 72 5f 35 32 33 31 37 5a 0f 30 78 37 35 |mber_52317Z.0x75| +000000e0 30 78 38 33 30 78 61 35 30 78 62 68 dd f5 01 70 |0x830xa50xbh...p| +000000f0 82 83 03 7a 04 35 32 32 64 |...z.522d| +000000f9 + +MESSAGE #1 AT 0x00000002 +a: "tags for first fixed value" +b: 1622559733 +c: 920 +n: B +d: E +e: "79034445678" +f: "250208889765444" +g: "35655678903421" +h: "79991232222" +i: 250 +j: 20 +k: 18122 +l: 22010 +m: "text for the first fixed value" +sub_2 { + a: 3 + b: "172.18.20.11" + c: 47855 + d: 32705 + e: 26855 + f: 51940 + g: "0x1dbb09" + h: "_49597" + i: "msc_number_52317" + j: "0x750x830xa50xb" + k: 31453 + l: 49538 + random_name: "522d" +} + +Binary representation is as expected + +tags for first fixed value 1622559733 920 1 79034445678 250208889765444 35655678903421 79991232222 250 20 18122 22010 text for the first fixed value \N \N \N \N \N \N \N \N \N \N \N 3 172.18.20.11 47855 32705 26855 51940 0x1dbb09 _49597 msc_number_52317 0x750x830xa50xb 31453 49538 1 522d +tags for first fixed value 1622559733 920 1 79034445678 250208889765444 35655678903421 79991232222 250 20 18122 22010 text for the first fixed value \N \N \N \N \N \N \N \N \N \N \N 3 172.18.20.11 47855 32705 26855 51940 0x1dbb09 _49597 msc_number_52317 0x750x830xa50xb 31453 49538 1 522d diff --git a/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh new file mode 100755 index 00000000000..ca915aca944 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -eo pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" + +# Check the output in the protobuf format +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_splitted_nested:some.Some" --input "$BINARY_FILE_PATH" + +# Check the input in the protobuf format (now the table contains the same data twice). +echo +$CLICKHOUSE_CLIENT --query "INSERT INTO splitted_nested_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_splitted_nested:Some'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM splitted_nested_protobuf_00825" + +rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE splitted_nested_protobuf_00825" diff --git a/tests/queries/0_stateless/format_schemas/00825_protobuf_format_splitted_nested.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_splitted_nested.proto new file mode 100644 index 00000000000..3157bfcd8c5 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_splitted_nested.proto @@ -0,0 +1,72 @@ +syntax = "proto3"; +package some; + +message Some { + + enum n_enum { + A = 0; + B = 1; + C = 2; + } + + enum d_enum { + D = 0; + E = 1; + F = 2; + G = 3; + H = 4; + I = 9; + K = 10; + } + string a = 1; + int64 b = 2; + int32 c = 3; + n_enum n = 4; + d_enum d = 5; + string e = 6; + string f = 7; + string g = 8; + string h = 9; + int32 i = 10; + int32 j = 11; + int32 k = 12; + int32 l = 13; + string m = 14; + + oneof sub { + SubOne sub_1 = 100; + SubTwo sub_2 = 101; + } +} + +message SubOne { + int32 a = 1; + int32 b = 2; + string c = 3; + string d = 4; + string e = 5; + string f = 6; + string g = 7; + string h = 8; + string i = 9; + string j = 10; + string k = 11; +} + +message SubTwo { + reserved 2,12; + + int32 a = 1; + string b = 3; + int32 c = 4; + int32 d = 5; + int32 e = 6; + int32 f = 7; + bytes g = 8; + string h = 9; + string i = 10; + string j = 11; + int64 k = 13; + int64 l = 14; + bytes random_name = 15; +} From 9e83275d28423a9ae62b7c2f8b806f6ef6244459 Mon Sep 17 00:00:00 2001 From: dankondr Date: Sun, 6 Jun 2021 17:52:08 +0300 Subject: [PATCH 079/491] Make string arrays constexpr --- src/Functions/dateName.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index d1551dd2442..be4888745fb 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -209,7 +209,7 @@ private: static inline void writeMonth(char *& target, Time source, const DateLUTImpl & timezone) { const auto month = ToMonthImpl::execute(source, timezone); - const String monthnames[12] + static constexpr std::string_view monthnames[] = {"January", "February", "March", @@ -243,7 +243,7 @@ private: static inline void writeWeekday(char *& target, Time source, const DateLUTImpl & timezone) { const auto day = ToDayOfWeekImpl::execute(source, timezone); - const String daynames[12] = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"}; + static constexpr std::string_view daynames[] = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"}; writeString(target, daynames[day - 1]); } @@ -262,7 +262,7 @@ private: writeNumber(target, ToSecondImpl::execute(source, timezone)); } - static inline void writeString(char *& target, const String & value) + static inline void writeString(char *& target, const std::string_view & value) { size_t size = value.size() + 1; /// With zero terminator memcpy(target, value.data(), size); From 6ba40c475af44f2af2f751e0e2cced19dcf608b9 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 7 Jun 2021 13:49:34 +0300 Subject: [PATCH 080/491] Code cleanup + comments. --- src/Disks/ReadIndirectBufferFromRemoteFS.cpp | 7 ++++--- src/Disks/S3/DiskS3.cpp | 1 + src/IO/ReadBufferFromS3.cpp | 9 +++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp index 2bbc2e2197e..960844e5281 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.cpp @@ -87,10 +87,7 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() { /// Find first available buffer that fits to given offset. if (!current_buf) - { current_buf = initialize(); - pos = working_buffer.begin(); - } /// If current buffer has remaining data - use it. if (current_buf) @@ -115,10 +112,14 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() template bool ReadIndirectBufferFromRemoteFS::nextAndShiftPosition() { + /// Transfer current position and working_buffer to actual ReadBuffer swap(*current_buf); + /// Position and working_buffer will be updated in next() call auto result = current_buf->next(); + /// and assigned to current buffer. swap(*current_buf); + /// absolute position is shifted by a data size that was read in next() call above. if (result) absolute_position += working_buffer.size(); diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 3ca8e07bba6..33c0c2a7f52 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -320,6 +320,7 @@ void DiskS3::startup() { auto settings = current_settings.get(); + /// Need to be enabled if it was disabled during shutdown() call. settings->client->EnableRequestProcessing(); if (!settings->send_metadata) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 86cf44178eb..0e342fb846a 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -91,18 +91,15 @@ bool ReadBufferFromS3::nextImpl() off_t ReadBufferFromS3::seek(off_t offset_, int whence) { + if (impl) + throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + if (whence != SEEK_SET) throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); if (offset_ < 0) throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); - if (impl) - { - impl.reset(); - pos = working_buffer.end(); - } - offset = offset_; return offset; From 7aa08a04b53bd508125f9022527ba46fb34932e1 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Mon, 7 Jun 2021 16:52:47 +0300 Subject: [PATCH 081/491] Fix unit-tests again --- src/Parsers/MySQL/ASTAlterCommand.cpp | 2 +- src/Parsers/MySQL/ASTAlterQuery.cpp | 2 +- src/Parsers/MySQL/ASTCreateQuery.cpp | 4 ++-- .../gtest_transform_query_for_external_database.cpp | 2 +- src/Storages/transformQueryForExternalDatabase.cpp | 9 ++------- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/Parsers/MySQL/ASTAlterCommand.cpp b/src/Parsers/MySQL/ASTAlterCommand.cpp index e0395c40fe5..b5b36ff3c74 100644 --- a/src/Parsers/MySQL/ASTAlterCommand.cpp +++ b/src/Parsers/MySQL/ASTAlterCommand.cpp @@ -242,7 +242,7 @@ static inline bool parseRenameCommand(IParser::Pos & pos, ASTPtr & node, Expecte } else if (ParserKeyword("TO").ignore(pos, expected) || ParserKeyword("AS").ignore(pos, expected)) { - if (!ParserCompoundIdentifier(false).parse(pos, new_name, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, new_name, expected)) return false; auto new_table_id = new_name->as()->getTableId(); diff --git a/src/Parsers/MySQL/ASTAlterQuery.cpp b/src/Parsers/MySQL/ASTAlterQuery.cpp index a6eb85e7472..59f0ada4e32 100644 --- a/src/Parsers/MySQL/ASTAlterQuery.cpp +++ b/src/Parsers/MySQL/ASTAlterQuery.cpp @@ -36,7 +36,7 @@ bool ParserAlterQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & e if (!ParserKeyword("ALTER TABLE").ignore(pos, expected)) return false; - if (!ParserCompoundIdentifier(false).parse(pos, table, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) return false; if (!ParserList(std::make_unique(), std::make_unique(TokenType::Comma)).parse(pos, command_list, expected)) diff --git a/src/Parsers/MySQL/ASTCreateQuery.cpp b/src/Parsers/MySQL/ASTCreateQuery.cpp index 22e13dbca54..227ac62f86d 100644 --- a/src/Parsers/MySQL/ASTCreateQuery.cpp +++ b/src/Parsers/MySQL/ASTCreateQuery.cpp @@ -68,14 +68,14 @@ bool ParserCreateQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (ParserKeyword("LIKE").ignore(pos, expected)) { - if (!ParserCompoundIdentifier(false).parse(pos, like_table, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, like_table, expected)) return false; } else if (ParserToken(TokenType::OpeningRoundBracket).ignore(pos, expected)) { if (ParserKeyword("LIKE").ignore(pos, expected)) { - if (!ParserCompoundIdentifier(false).parse(pos, like_table, expected)) + if (!ParserCompoundIdentifier(true).parse(pos, like_table, expected)) return false; if (!ParserToken(TokenType::ClosingRoundBracket).ignore(pos, expected)) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index e61123a3aa8..1d4cad576fa 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -176,7 +176,7 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))"); check(state, 1, "SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42", - R"(SELECT "column" FROM "test"."table" WHERE ("column" = 42))"); + R"(SELECT "column" FROM "test"."table" WHERE "column" = 42)"); } TEST(TransformQueryForExternalDatabase, Issue7245) diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 7a07cd94662..4e299c5a357 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -273,20 +273,15 @@ String transformQueryForExternalDatabase( { if (function->name == "and") { - bool compatible_found = false; auto new_function_and = makeASTFunction("and"); for (const auto & elem : function->arguments->children) { if (isCompatible(*elem)) - { new_function_and->arguments->children.push_back(elem); - compatible_found = true; - } } if (new_function_and->arguments->children.size() == 1) - new_function_and->name = ""; - - if (compatible_found) + select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(new_function_and->arguments->children[0])); + else if (new_function_and->arguments->children.size() > 1) select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(new_function_and)); } } From 483d415b24077590c12d5477484ecd4938fe00bc Mon Sep 17 00:00:00 2001 From: George Date: Mon, 7 Jun 2021 21:24:26 +0300 Subject: [PATCH 082/491] First draft --- docs/en/sql-reference/data-types/array.md | 21 +++++++++++++++ .../nested-data-structures/nested.md | 2 +- docs/en/sql-reference/data-types/nullable.md | 21 +++++++++++++++ docs/en/sql-reference/data-types/tuple.md | 26 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 41e35aaa96f..ced7e399f45 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -74,4 +74,25 @@ Received exception from server (version 1.1.54388): Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. ``` +It is possible to use `size0` subcolumns that can be read without reading the whole column: + +```sql +CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_arr VALUES ([1]) ([]) ([1, 2, 3]) ([1, 2]); + +SELECT a.size0 FROM t_arr; +``` + +Result: + +``` text +┌─a.size0─┐ +│ 1 │ +│ 0 │ +│ 3 │ +│ 2 │ +└─────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/data_types/array/) diff --git a/docs/en/sql-reference/data-types/nested-data-structures/nested.md b/docs/en/sql-reference/data-types/nested-data-structures/nested.md index 302d41c1357..902ae8ef715 100644 --- a/docs/en/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/en/sql-reference/data-types/nested-data-structures/nested.md @@ -34,7 +34,7 @@ CREATE TABLE test.visits This example declares the `Goals` nested data structure, which contains data about conversions (goals reached). Each row in the ‘visits’ table can correspond to zero or any number of conversions. -Only a single nesting level is supported. Columns of nested structures containing arrays are equivalent to multidimensional arrays, so they have limited support (there is no support for storing these columns in tables with the MergeTree engine). +Arbitrary levels of nesting are supported. Columns of nested structures containing arrays are equivalent to multidimensional arrays, so they have limited support (there is no support for storing these columns in tables with the MergeTree engine). In most cases, when working with a nested data structure, its columns are specified with column names separated by a dot. These columns make up an array of matching types. All the column arrays of a single nested data structure have the same length. diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 4207e389734..247129a0cf3 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -41,4 +41,25 @@ SELECT x + y FROM t_null └────────────┘ ``` +It is possible to use `null` subcolumns that can be read without reading the whole column: + +``` sql +CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); + +SELECT n.null FROM nullable; +``` + +Result: + +``` text +┌─n.null─┐ +│ 0 │ +│ 1 │ +│ 0 │ +│ 1 │ +└────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/data_types/nullable/) diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index e396006d957..21b4cc7e0d8 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -47,4 +47,30 @@ SELECT tuple(1, NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` +It is possible to read elements of named tuples using indexes and names: + +``` sql +CREATE TABLE named_tuples (`a` Tuple(s String, i Int64)) ENGINE = Memory; + +INSERT INTO named_tuples VALUES (('y', 10)), (('x',-10)); + +SELECT a.s FROM named_tuples; + +SELECT a.2 FROM named_tuples; +``` + +Result: + +``` text +┌─a.s─┐ +│ y │ +│ x │ +└─────┘ + +┌─tupleElement(a, 2)─┐ +│ 10 │ +│ -10 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.tech/docs/en/data_types/tuple/) From fb987cf2c7732634b17c96b97e531bfcbcc63966 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 8 Jun 2021 15:00:28 +0300 Subject: [PATCH 083/491] Fix space in ReadBufferFromFileDecorator.cpp --- src/IO/ReadBufferFromFileDecorator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadBufferFromFileDecorator.cpp b/src/IO/ReadBufferFromFileDecorator.cpp index 53fde7789bc..5810eccbac7 100644 --- a/src/IO/ReadBufferFromFileDecorator.cpp +++ b/src/IO/ReadBufferFromFileDecorator.cpp @@ -13,7 +13,7 @@ ReadBufferFromFileDecorator::ReadBufferFromFileDecorator(std::unique_ptr(impl.get())) + if (ReadBufferFromFileBase * buffer = dynamic_cast(impl.get())) return buffer->getFileName(); return std::string(); } From 3ade38df82cca68e237d456f54fc9bdd61fa38e8 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 8 Jun 2021 22:11:22 +0300 Subject: [PATCH 084/491] remove copypaste --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 15 +++++ src/Storages/MergeTree/IMergeTreeDataPart.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 55 +------------------ src/Storages/StorageReplicatedMergeTree.h | 2 - 5 files changed, 20 insertions(+), 57 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 703cf32f743..ebc75d63b75 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1097,6 +1097,21 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ } +void IMergeTreeDataPart::removeIfNotLockedInS3() const +{ + try + { + /// TODO Unlocking in try-catch looks ugly. Special "keep_s3" flag + /// which is a bit different from "keep_s3_on_delete" flag looks ugly too. + bool keep_s3 = !storage.unlockSharedData(*this); + remove(keep_s3); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__, "There is a problem with deleting part " + name + " from filesystem"); + } +} + void IMergeTreeDataPart::remove(bool keep_s3) const { if (!isStoredOnDisk()) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 53640b41507..59c19349e8f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -127,6 +127,7 @@ public: void assertOnDisk() const; void remove(bool keep_s3 = false) const; + void removeIfNotLockedInS3() const; void projectionRemove(const String & parent_to, bool keep_s3 = false) const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6eeabd9604d..628e52f079d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1313,7 +1313,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re CurrentThread::attachTo(thread_group); LOG_DEBUG(log, "Removing part from filesystem {}", part->name); - part->remove(); + part->removeIfNotLockedInS3(); }); } @@ -1324,7 +1324,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re for (const DataPartPtr & part : parts_to_remove) { LOG_DEBUG(log, "Removing part from filesystem {}", part->name); - part->remove(); + part->removeIfNotLockedInS3(); } } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ea81f64659f..1e5c487e424 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5939,57 +5939,6 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio return CancellationCode::CancelSent; } -void StorageReplicatedMergeTree::removePartsFromFilesystem(const DataPartsVector & parts) -{ - auto remove_part = [&](const auto & part) - { - LOG_DEBUG(log, "Removing part from filesystem {}", part.name); - try - { - bool keep_s3 = !this->unlockSharedData(part); - part.remove(keep_s3); - } - catch (...) - { - tryLogCurrentException(log, "There is a problem with deleting part " + part.name + " from filesystem"); - } - }; - - const auto settings = getSettings(); - if (settings->max_part_removal_threads > 1 && parts.size() > settings->concurrent_part_removal_threshold) - { - /// Parallel parts removal. - - size_t num_threads = std::min(settings->max_part_removal_threads, parts.size()); - ThreadPool pool(num_threads); - - /// NOTE: Under heavy system load you may get "Cannot schedule a task" from ThreadPool. - for (const DataPartPtr & part : parts) - { - pool.scheduleOrThrowOnError([&, thread_group = CurrentThread::getGroup()] - { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachTo(thread_group); - - remove_part(*part); - }); - } - - pool.wait(); - } - else - { - for (const DataPartPtr & part : parts) - { - remove_part(*part); - } - } -} - void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() { auto table_lock = lockForShare( @@ -6017,7 +5966,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() /// Delete duplicate parts from filesystem if (!parts_to_delete_only_from_filesystem.empty()) { - removePartsFromFilesystem(parts_to_delete_only_from_filesystem); + clearPartsFromFilesystem(parts_to_delete_only_from_filesystem); removePartsFinally(parts_to_delete_only_from_filesystem); LOG_DEBUG(log, "Removed {} old duplicate parts", parts_to_delete_only_from_filesystem.size()); @@ -6062,7 +6011,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() /// Remove parts from filesystem and finally from data_parts if (!parts_to_remove_from_filesystem.empty()) { - removePartsFromFilesystem(parts_to_remove_from_filesystem); + clearPartsFromFilesystem(parts_to_remove_from_filesystem); removePartsFinally(parts_to_remove_from_filesystem); LOG_DEBUG(log, "Removed {} old parts", parts_to_remove_from_filesystem.size()); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 8ffb4974cb3..2ae19387ee8 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -439,8 +439,6 @@ private: /// Just removes part from ZooKeeper using previous method void removePartFromZooKeeper(const String & part_name); - void removePartsFromFilesystem(const DataPartsVector & parts); - /// Quickly removes big set of parts from ZooKeeper (using async multi queries) void removePartsFromZooKeeper(zkutil::ZooKeeperPtr & zookeeper, const Strings & part_names, NameSet * parts_should_be_retried = nullptr); From 18e8f0eb5e51f2851e4e0a04282cb1d60a2a21c5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 4 Jun 2021 09:43:56 +0300 Subject: [PATCH 085/491] Add ability to push down LIMIT for distributed queries This way the remote nodes will not need to send all the rows, so this will decrease network io and also this will make queries w/ optimize_aggregation_in_order=1/LIMIT X and w/o ORDER BY faster since it initiator will not need to read all the rows, only first X (but note that for this you need to your data to be sharded correctly or you may get inaccurate results). Note, that having lots of processing stages will increase the complexity of interpreter (it is already not that clean and simple right now). Although using separate QueryProcessingStage looks pretty natural. Another option is to make WithMergeableStateAfterAggregation always, but in this case you will not be able to disable only this optimization, i.e. if there will be some issue with it. v2: fix OFFSET v3: convert 01814_distributed_push_down_limit test to .sh and add retries v4: add test with OFFSET v5: add new query stage into the bash completion v6/tests: use LIMIT O,L syntax over LIMIT L OFFSET O since it is broken in ANTLR parser https://clickhouse-test-reports.s3.yandex.net/23027/a18a06399b7aeacba7c50b5d1e981ada5df19745/functional_stateless_tests_(antlr_debug).html#fail1 v7/tests: set use_hedged_requests to 0, to avoid excessive log entries on retries https://clickhouse-test-reports.s3.yandex.net/23027/a18a06399b7aeacba7c50b5d1e981ada5df19745/functional_stateless_tests_flaky_check_(address).html#fail1 --- docs/en/operations/settings/settings.md | 12 ++ .../completions/clickhouse-bootstrap | 1 + programs/benchmark/Benchmark.cpp | 2 +- programs/client/Client.cpp | 2 +- src/Core/QueryProcessingStage.cpp | 2 + src/Core/QueryProcessingStage.h | 10 +- src/Core/Settings.h | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 22 +-- .../getHeaderForProcessingStage.cpp | 1 + src/Storages/StorageDistributed.cpp | 19 +- ...1814_distributed_push_down_limit.reference | 37 ++++ .../01814_distributed_push_down_limit.sh | 167 ++++++++++++++++++ ...tate_after_aggregation_and_limit.reference | 1 + ...eable_state_after_aggregation_and_limit.sh | 8 + .../queries/0_stateless/arcadia_skip_list.txt | 1 + 15 files changed, 270 insertions(+), 16 deletions(-) create mode 100644 tests/queries/0_stateless/01814_distributed_push_down_limit.reference create mode 100755 tests/queries/0_stateless/01814_distributed_push_down_limit.sh create mode 100644 tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.reference create mode 100755 tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.sh diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 10461eacbff..59417753db0 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1578,6 +1578,18 @@ FORMAT PrettyCompactMonoBlock Default value: 0 +## distributed_push_down_limit (#distributed-push-down-limit} + +LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT. + +Possible values: + +- 0 - Disabled +- 1 - Enabled + +!!! note "Note" + That with this setting the result of the query may be inaccurate. + ## optimize_skip_unused_shards_limit {#optimize-skip-unused-shards-limit} Limit for number of sharding key values, turns off `optimize_skip_unused_shards` if the limit is reached. diff --git a/programs/bash-completion/completions/clickhouse-bootstrap b/programs/bash-completion/completions/clickhouse-bootstrap index 7109148a192..793d47501dd 100644 --- a/programs/bash-completion/completions/clickhouse-bootstrap +++ b/programs/bash-completion/completions/clickhouse-bootstrap @@ -20,6 +20,7 @@ CLICKHOUSE_QueryProcessingStage=( fetch_columns with_mergeable_state with_mergeable_state_after_aggregation + with_mergeable_state_after_aggregation_and_limit ) CLICKHOUSE_Format=( diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 2e48c5d20c5..c8f1a4eef47 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -580,7 +580,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("query", value()->default_value(""), "query to execute") ("concurrency,c", value()->default_value(1), "number of parallel queries") ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("stage", value()->default_value("complete"), "request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation") + ("stage", value()->default_value("complete"), "request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit") ("iterations,i", value()->default_value(0), "amount of queries to be executed") ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") ("randomize,r", value()->default_value(false), "randomize order of execution") diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index f268d2b5cdc..2000f04877f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2463,7 +2463,7 @@ public: ("password", po::value()->implicit_value("\n", ""), "password") ("ask-password", "ask-password") ("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server") - ("stage", po::value()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation") + ("stage", po::value()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit") ("query_id", po::value(), "query_id") ("query,q", po::value(), "query") ("database,d", po::value(), "database") diff --git a/src/Core/QueryProcessingStage.cpp b/src/Core/QueryProcessingStage.cpp index 14bde0e548d..b5b837e1f61 100644 --- a/src/Core/QueryProcessingStage.cpp +++ b/src/Core/QueryProcessingStage.cpp @@ -24,6 +24,8 @@ namespace QueryProcessingStage stage = WithMergeableState; else if (stage_string == "with_mergeable_state_after_aggregation") stage = WithMergeableStateAfterAggregation; + else if (stage_string == "with_mergeable_state_after_aggregation_and_limit") + stage = WithMergeableStateAfterAggregationAndLimit; else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown query processing stage: {}", stage_string); diff --git a/src/Core/QueryProcessingStage.h b/src/Core/QueryProcessingStage.h index 01e7e12ab1e..7ccaa17eaed 100644 --- a/src/Core/QueryProcessingStage.h +++ b/src/Core/QueryProcessingStage.h @@ -26,8 +26,15 @@ namespace QueryProcessingStage /// It is used for auto distributed_group_by_no_merge optimization for distributed engine. /// (See comments in StorageDistributed). WithMergeableStateAfterAggregation = 3, + /// Same as WithMergeableStateAfterAggregation but also will apply limit on each shard. + /// + /// This query stage will be used for auto + /// distributed_group_by_no_merge/distributed_push_down_limit + /// optimization. + /// (See comments in StorageDistributed). + WithMergeableStateAfterAggregationAndLimit = 4, - MAX = 4, + MAX = 5, }; inline const char * toString(UInt64 stage) @@ -38,6 +45,7 @@ namespace QueryProcessingStage "WithMergeableState", "Complete", "WithMergeableStateAfterAggregation", + "WithMergeableStateAfterAggregationAndLimit", }; return stage < MAX ? data[stage] diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 08b69e52bbc..fb3ba58733b 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -118,6 +118,7 @@ class IColumn; \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ + M(UInt64, distributed_push_down_limit, 0, "If 1, LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 0f8f381cea7..28d4c425f1a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -632,7 +632,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() /// Running on the initiating server during distributed processing or if query is not distributed. /// /// Also note that with distributed_group_by_no_merge=2 (i.e. when optimize_distributed_group_by_sharding_key takes place) - /// the query on the remote server will be processed up to WithMergeableStateAfterAggregation, + /// the query on the remote server will be processed up to WithMergeableStateAfterAggregationAndLimit, /// So it will do partial second stage (second_stage=true), and initiator will do the final part. bool second_stage = from_stage <= QueryProcessingStage::WithMergeableState && options.to_stage > QueryProcessingStage::WithMergeableState; @@ -704,7 +704,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() return res; } - if (options.to_stage == QueryProcessingStage::Enum::WithMergeableStateAfterAggregation) + if (options.to_stage >= QueryProcessingStage::Enum::WithMergeableStateAfterAggregation) { // It's different from selected_columns, see the comment above for // WithMergeableState stage. @@ -1011,10 +1011,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu /// Support optimize_distributed_group_by_sharding_key /// Is running on the initiating server during distributed processing? - if (from_stage == QueryProcessingStage::WithMergeableStateAfterAggregation) + if (from_stage >= QueryProcessingStage::WithMergeableStateAfterAggregation) from_aggregation_stage = true; /// Is running on remote servers during distributed processing? - if (options.to_stage == QueryProcessingStage::WithMergeableStateAfterAggregation) + if (options.to_stage >= QueryProcessingStage::WithMergeableStateAfterAggregation) to_aggregation_stage = true; /// Read the data from Storage. from_stage - to what stage the request was completed in Storage. @@ -1300,7 +1300,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu * but there is no aggregation, then on the remote servers ORDER BY was made * - therefore, we merge the sorted streams from remote servers. * - * Also in case of remote servers was process the query up to WithMergeableStateAfterAggregation + * Also in case of remote servers was process the query up to WithMergeableStateAfterAggregationAndLimit * (distributed_group_by_no_merge=2 or optimize_distributed_group_by_sharding_key=1 takes place), * then merge the sorted streams is enough, since remote servers already did full ORDER BY. */ @@ -1334,13 +1334,15 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu } } + bool apply_limit = options.to_stage != QueryProcessingStage::WithMergeableStateAfterAggregation; + bool apply_offset = options.to_stage != QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; bool has_prelimit = false; - if (!to_aggregation_stage && + if (apply_limit && query.limitLength() && !query.limit_with_ties && !hasWithTotalsInAnySubqueryInFromClause(query) && !query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes && !has_withfill) { - executePreLimit(query_plan, false); + executePreLimit(query_plan, /* do_not_skip_offset= */!apply_offset); has_prelimit = true; } @@ -1367,7 +1369,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu } /// Projection not be done on the shards, since then initiator will not find column in blocks. - /// (significant only for WithMergeableStateAfterAggregation). + /// (significant only for WithMergeableStateAfterAggregation/WithMergeableStateAfterAggregationAndLimit). if (!to_aggregation_stage) { /// We must do projection after DISTINCT because projection may remove some columns. @@ -1378,10 +1380,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu executeExtremes(query_plan); /// Limit is no longer needed if there is prelimit. - if (!to_aggregation_stage && !has_prelimit) + if (apply_limit && !has_prelimit) executeLimit(query_plan); - if (!to_aggregation_stage) + if (apply_offset) executeOffset(query_plan); } } diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index 9c7c86a0b88..e77b64dea48 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -69,6 +69,7 @@ Block getHeaderForProcessingStage( case QueryProcessingStage::WithMergeableState: case QueryProcessingStage::Complete: case QueryProcessingStage::WithMergeableStateAfterAggregation: + case QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit: case QueryProcessingStage::MAX: { auto query = query_info.query->clone(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index ace0963bc0a..a1aacf95105 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -288,6 +288,7 @@ void replaceConstantExpressions( /// is one of the following: /// - QueryProcessingStage::Complete /// - QueryProcessingStage::WithMergeableStateAfterAggregation +/// - QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit /// - none (in this case regular WithMergeableState should be used) std::optional getOptimizedQueryProcessingStage(const SelectQueryInfo & query_info, bool extremes, const Block & sharding_key_block) { @@ -349,13 +350,13 @@ std::optional getOptimizedQueryProcessingStage(const // ORDER BY const ASTPtr order_by = select.orderBy(); if (order_by) - return QueryProcessingStage::WithMergeableStateAfterAggregation; + return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; // LIMIT BY // LIMIT // OFFSET if (select.limitBy() || select.limitLength() || select.limitOffset()) - return QueryProcessingStage::WithMergeableStateAfterAggregation; + return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; // Only simple SELECT FROM GROUP BY sharding_key can use Complete state. return QueryProcessingStage::Complete; @@ -514,11 +515,23 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( if (settings.distributed_group_by_no_merge) { if (settings.distributed_group_by_no_merge == DISTRIBUTED_GROUP_BY_NO_MERGE_AFTER_AGGREGATION) - return QueryProcessingStage::WithMergeableStateAfterAggregation; + { + if (settings.distributed_push_down_limit) + return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; + else + return QueryProcessingStage::WithMergeableStateAfterAggregation; + } else + { + /// NOTE: distributed_group_by_no_merge=1 does not respect distributed_push_down_limit + /// (since in this case queries processed separatelly and the initiator is just a proxy in this case). return QueryProcessingStage::Complete; + } } + if (settings.distributed_push_down_limit) + return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; + /// Nested distributed query cannot return Complete stage, /// since the parent query need to aggregate the results after. if (to_stage == QueryProcessingStage::WithMergeableState) diff --git a/tests/queries/0_stateless/01814_distributed_push_down_limit.reference b/tests/queries/0_stateless/01814_distributed_push_down_limit.reference new file mode 100644 index 00000000000..f879f2cbd21 --- /dev/null +++ b/tests/queries/0_stateless/01814_distributed_push_down_limit.reference @@ -0,0 +1,37 @@ +distributed_push_down_limit=0 +100 100 +distributed_push_down_limit=1 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +40 40 +auto-distributed_push_down_limit +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +40 40 +distributed_push_down_limit=1 with OFFSET +97 +96 +96 +95 +95 +94 +94 +93 +93 +92 diff --git a/tests/queries/0_stateless/01814_distributed_push_down_limit.sh b/tests/queries/0_stateless/01814_distributed_push_down_limit.sh new file mode 100755 index 00000000000..93321646037 --- /dev/null +++ b/tests/queries/0_stateless/01814_distributed_push_down_limit.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2206 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# -- NOTE: this test cannot use 'current_database = $CLICKHOUSE_DATABASE', +# -- because it does not propagated via remote queries, +# -- hence it uses query_id/initial_query_id. + +function setup() +{ + $CLICKHOUSE_CLIENT -nm -q " + drop table if exists data_01814; + drop table if exists dist_01814; + + create table data_01814 (key Int) Engine=MergeTree() order by key settings index_granularity=10 as select * from numbers(100); + create table dist_01814 as data_01814 engine=Distributed('test_cluster_two_shards', $CLICKHOUSE_DATABASE, data_01814, key); + " +} + +function cleanup() +{ + $CLICKHOUSE_CLIENT -nm -q " + drop table data_01814; + drop table dist_01814; + " +} + +function make_query_id() +{ + echo "$(tr -cd '[:lower:]' < /dev/urandom | head -c10)-$CLICKHOUSE_DATABASE" +} + +function test_distributed_push_down_limit_with_query_log() +{ + local table=$1 && shift + local offset=$1 && shift + local query_id + + query_id="$(make_query_id)" + + # NOTES: + # - max_rows_to_read_leaf cannot be used since it does not know anything + # about optimize_aggregation_in_order, + # - limit push down can be checked only with optimize_aggregation_in_order, + # since otherwise the query will be canceled too early, and read_rows will be + # small. + local settings_and_opts=( + --query_id "$query_id" + + --max_block_size 20 + --optimize_aggregation_in_order 1 + --log_queries 1 + --log_queries_min_type 'QUERY_FINISH' + + # disable hedged requests to avoid excessive log entries + --use_hedged_requests 0 + + "$@" + ) + + $CLICKHOUSE_CLIENT "${settings_and_opts[@]}" -q "select * from $table group by key limit $offset, 10" + + $CLICKHOUSE_CLIENT -nm -q " + system flush logs; + select read_rows from system.query_log + where + event_date = today() + and query_kind = 'Select' /* exclude DESC TABLE */ + and initial_query_id = '$query_id' and initial_query_id != query_id; + " | xargs # convert new lines to spaces +} + +function test_distributed_push_down_limit_0() +{ + local args=( + "remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814)" + 0 # offset + --distributed_push_down_limit 0 + ) + test_distributed_push_down_limit_with_query_log "${args[@]}" "$@" +} + +function test_distributed_push_down_limit_1() +{ + local args=( + "remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814)" + 0 # offset + --distributed_push_down_limit 1 + ) + test_distributed_push_down_limit_with_query_log "${args[@]}" +} + +function test_distributed_push_down_limit_1_offset() +{ + local settings_and_opts=( + --distributed_push_down_limit 1 + ) + + $CLICKHOUSE_CLIENT "${settings_and_opts[@]}" -q "select * from remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814) group by key order by key desc limit 5, 10" +} + +function test_auto_distributed_push_down_limit() +{ + local args=( + dist_01814 + 0 # offset + --optimize_skip_unused_shards 1 + --optimize_distributed_group_by_sharding_key 1 + --prefer_localhost_replica 0 + --distributed_push_down_limit 0 + ) + test_distributed_push_down_limit_with_query_log "${args[@]}" +} + +function main() +{ + setup + trap cleanup EXIT + + echo 'distributed_push_down_limit=0' + test_distributed_push_down_limit_0 --format Null + + # + # The following tests (tests with distributed_push_down_limit=1) requires + # retries, since the query may be canceled earlier due to LIMIT, and so + # only one shard will be processed, and it will get not 40 but 20 rows: + # + # 1.160920 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} executeQuery: (from [::ffff:127.0.0.1]:42778, initial_query_id: 66cf643c-b1b4-4f7e-942a-c4c3493029f6, using production parser) (comment: /usr/share/clickhouse-test/queries/0_stateless/01814_distributed_push_down_limit.sql) WITH CAST('test_31uut9', 'String') AS id_distributed_push_down_limit_1 SELECT key FROM test_31uut9.data_01814 GROUP BY key LIMIT 10 + # 1.214964 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} ContextAccess (default): Access granted: SELECT(key) ON test_31uut9.data_01814 + # 1.216790 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} test_31uut9.data_01814 (b484ad2e-0591-4faf-8110-1dcbd7cdd0db) (SelectExecutor): Key condition: unknown + # 1.227245 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} test_31uut9.data_01814 (b484ad2e-0591-4faf-8110-1dcbd7cdd0db) (SelectExecutor): Selected 1/1 parts by partition key, 1 parts by primary key, 10/11 marks by primary key, 10 marks to read from 1 ranges + # 1.228452 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} MergeTreeSelectProcessor: Reading 3 ranges from part all_1_1_0, approx. 100 rows starting from 0 + # 1.229104 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} InterpreterSelectQuery: FetchColumns -> WithMergeableStateAfterAggregationAndLimit + # 1.339085 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} TCPHandler: Query was cancelled. + # 1.416573 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} executeQuery: Read 20 rows, 80.00 B in 0.254374666 sec., 78 rows/sec., 314.50 B/sec. + # 1.419006 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} MemoryTracker: Peak memory usage (for query): 0.00 B. + # + + local out out_lines max_tries=20 + + echo 'distributed_push_down_limit=1' + for ((i = 0; i < max_tries; ++i)); do + out=$(test_distributed_push_down_limit_1) + out_lines=( $out ) + if [[ ${#out_lines[@]} -gt 2 ]] && [[ ${out_lines[-1]} = 40 ]] && [[ ${out_lines[-2]} = 40 ]]; then + break + fi + done + echo "$out" + + echo 'auto-distributed_push_down_limit' + for ((i = 0; i < max_tries; ++i)); do + out=$(test_auto_distributed_push_down_limit) + out_lines=( $out ) + if [[ ${#out_lines[@]} -gt 2 ]] && [[ ${out_lines[-1]} = 40 ]] && [[ ${out_lines[-2]} = 40 ]]; then + break + fi + done + echo "$out" + + echo 'distributed_push_down_limit=1 with OFFSET' + test_distributed_push_down_limit_1_offset +} +main "$@" diff --git a/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.reference b/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.sh b/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.sh new file mode 100755 index 00000000000..0efacc4ac31 --- /dev/null +++ b/tests/queries/0_stateless/01815_with_mergeable_state_after_aggregation_and_limit.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# with_mergeable_state_after_aggregation will not stop after 1 row, while with_mergeable_state_after_aggregation_and_limit should +$CLICKHOUSE_CLIENT -q 'select * from system.numbers limit 1' --stage with_mergeable_state_after_aggregation_and_limit diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index dac43ff5d4b..5f6ec299888 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -234,6 +234,7 @@ 01801_distinct_group_by_shard 01804_dictionary_decimal256_type 01801_s3_distributed +01814_distributed_push_down_limit 01833_test_collation_alvarotuso 01850_dist_INSERT_preserve_error 01870_modulo_partition_key From cef22688ff00bde181317adc567e2000fa58379d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 9 Jun 2021 15:36:47 +0300 Subject: [PATCH 086/491] make code less ugly --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 47 ++++++++++++------- src/Storages/MergeTree/IMergeTreeDataPart.h | 7 +-- src/Storages/MergeTree/MergeTreeData.cpp | 13 +++-- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index ebc75d63b75..2dfc3ff7a2d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -429,9 +429,14 @@ void IMergeTreeDataPart::removeIfNeeded() } if (parent_part) - projectionRemove(parent_part->getFullRelativePath(), keep_s3_on_delete); + { + std::optional keep_shared_data = keepSharedDataInDecoupledStorage(); + if (!keep_shared_data.has_value()) + return; + projectionRemove(parent_part->getFullRelativePath(), *keep_shared_data); + } else - remove(keep_s3_on_delete); + remove(); if (state == State::DeleteOnDestroy) { @@ -1096,24 +1101,30 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ storage.lockSharedData(*this); } - -void IMergeTreeDataPart::removeIfNotLockedInS3() const +std::optional IMergeTreeDataPart::keepSharedDataInDecoupledStorage() const { + /// NOTE: It's needed for S3 zero-copy replication + if (force_keep_shared_data) + return true; + + /// TODO Unlocking in try-catch and ignoring exception look ugly try { - /// TODO Unlocking in try-catch looks ugly. Special "keep_s3" flag - /// which is a bit different from "keep_s3_on_delete" flag looks ugly too. - bool keep_s3 = !storage.unlockSharedData(*this); - remove(keep_s3); + return !storage.unlockSharedData(*this); } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__, "There is a problem with deleting part " + name + " from filesystem"); } + return {}; } -void IMergeTreeDataPart::remove(bool keep_s3) const +void IMergeTreeDataPart::remove() const { + std::optional keep_shared_data = keepSharedDataInDecoupledStorage(); + if (!keep_shared_data.has_value()) + return; + if (!isStoredOnDisk()) return; @@ -1123,7 +1134,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (isProjectionPart()) { LOG_WARNING(storage.log, "Projection part {} should be removed by its parent {}.", name, parent_part->name); - projectionRemove(parent_part->getFullRelativePath(), keep_s3); + projectionRemove(parent_part->getFullRelativePath(), *keep_shared_data); return; } @@ -1149,7 +1160,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_WARNING(storage.log, "Directory {} (to which part must be renamed before removing) already exists. Most likely this is due to unclean restart. Removing it.", fullPath(disk, to)); try { - disk->removeSharedRecursive(fs::path(to) / "", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", *keep_shared_data); } catch (...) { @@ -1176,7 +1187,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const std::unordered_set projection_directories; for (const auto & [p_name, projection_part] : projection_parts) { - projection_part->projectionRemove(to, keep_s3); + projection_part->projectionRemove(to, *keep_shared_data); projection_directories.emplace(p_name + ".proj"); } @@ -1184,7 +1195,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(fs::path(to) / "", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", *keep_shared_data); } else { @@ -1199,17 +1210,17 @@ void IMergeTreeDataPart::remove(bool keep_s3) const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(fs::path(to) / file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(fs::path(to) / file, keep_s3); + disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, *keep_shared_data); + disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, *keep_shared_data); disk->removeDirectory(to); } @@ -1219,7 +1230,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(fs::path(to) / "", keep_s3); + disk->removeSharedRecursive(fs::path(to) / "", *keep_shared_data); } } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 59c19349e8f..e05d0c5f487 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -126,8 +126,7 @@ public: /// Throws an exception if part is not stored in on-disk format. void assertOnDisk() const; - void remove(bool keep_s3 = false) const; - void removeIfNotLockedInS3() const; + void remove() const; void projectionRemove(const String & parent_to, bool keep_s3 = false) const; @@ -199,7 +198,7 @@ public: mutable std::atomic is_frozen {false}; /// Flag for keep S3 data when zero-copy replication over S3 turned on. - mutable bool keep_s3_on_delete = false; + mutable bool force_keep_shared_data = false; /** * Part state is a stage of its lifetime. States are ordered and state of a part could be increased only. @@ -426,6 +425,8 @@ protected: String getRelativePathForDetachedPart(const String & prefix) const; + std::optional keepSharedDataInDecoupledStorage() const; + private: /// In compact parts order of columns is necessary NameToNumber column_name_to_position; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 628e52f079d..b72b577adce 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1313,7 +1313,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re CurrentThread::attachTo(thread_group); LOG_DEBUG(log, "Removing part from filesystem {}", part->name); - part->removeIfNotLockedInS3(); + part->remove(); }); } @@ -1324,7 +1324,7 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re for (const DataPartPtr & part : parts_to_remove) { LOG_DEBUG(log, "Removing part from filesystem {}", part->name); - part->removeIfNotLockedInS3(); + part->remove(); } } } @@ -2736,17 +2736,16 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) /// We do not check allow_s3_zero_copy_replication here because data may be shared /// when allow_s3_zero_copy_replication turned on and off again - original_active_part->keep_s3_on_delete = false; + original_active_part->force_keep_shared_data = false; if (original_active_part->volume->getDisk()->getType() == DiskType::Type::S3) { if (part_copy->volume->getDisk()->getType() == DiskType::Type::S3 && original_active_part->getUniqueId() == part_copy->getUniqueId()) - { /// May be when several volumes use the same S3 storage - original_active_part->keep_s3_on_delete = true; + { + /// May be when several volumes use the same S3 storage + original_active_part->force_keep_shared_data = true; } - else - original_active_part->keep_s3_on_delete = !unlockSharedData(*original_active_part); } modifyPartState(original_active_part, DataPartState::DeleteOnDestroy); From 3ba808a7409fe0cdaa606020b876881a40d0d5ad Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 9 Jun 2021 21:19:34 +0300 Subject: [PATCH 087/491] Update formats.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавил таблицу соответствия типов для Arrow. --- docs/en/interfaces/formats.md | 56 +++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 4dc0fa1cff0..b36e9637179 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1249,6 +1249,9 @@ The table below shows supported data types and how they match ClickHouse [data t | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | | — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Elements of value `Array` type can be array or `Nullable` value. ClickHouse supports configurable precision of `Decimal` type. The `INSERT` query treats the Parquet `DECIMAL` type as the ClickHouse `Decimal128` type. @@ -1276,7 +1279,54 @@ To exchange data with Hadoop, you can use [HDFS table engine](../engines/table-e [Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats. ClickHouse supports read and write operations for these formats. -`Arrow` is Apache Arrow’s “file mode” format. It is designed for in-memory random access. +`Arrow` is Apache Arrow’s "file mode" format. It is designed for in-memory random access. + +### Data Types Matching {#data_types-matching-4} + +The table below shows supported data types and how they match ClickHouse [data types](../sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. + +| Arrow data type (`INSERT`) | ClickHouse data type | Arrow data type (`SELECT`) | +|----------------------------|-----------------------------------------------------|----------------------------| +| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | `UINT8` | +| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | `INT8` | +| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | `UINT16` | +| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | `INT16` | +| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | `UINT32` | +| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | `INT32` | +| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | `UINT64` | +| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | `INT64` | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT32` | +| `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `FLOAT64` | +| `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `UTF8` | +| `STRING` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | +| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `-` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Elements of value `Array` type can be array or value of the `Nullable` type. + +ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the Arrow `DECIMAL` type as the ClickHouse `Decimal128` type. + +Unsupported Arrow data types: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. + +The data types of ClickHouse table columns do not have to match the corresponding Arrow data fields. When inserting data, ClickHouse interprets data types according to the table above and then [casts](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) the data to the data type set for the ClickHouse table column. + +### Inserting Data {#inserting-data-3} + +You can insert Arrow data from a file into ClickHouse table by the following command: + +``` bash +$ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" +``` + +### Selecting Data {#selecting-data-3} + +You can select data from a ClickHouse table and save them into some file in the Arrow format by the following command: + +``` bash +$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow} +``` ## ArrowStream {#data-format-arrow-stream} @@ -1306,7 +1356,9 @@ The table below shows supported data types and how they match ClickHouse [data t | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | -| `-` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Elements of value `Array` type can be array or value of the `Nullable` type. ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the ORC `DECIMAL` type as the ClickHouse `Decimal128` type. From d52b87d47d95d20b5ca762a884414eb4ec577e50 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 10 Jun 2021 12:37:33 +0300 Subject: [PATCH 088/491] Always detach proken parts with wrong partition id. --- src/Storages/MergeTree/MergeTreeData.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index acf362e5b1f..1185e7e2383 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -123,6 +123,7 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int SUPPORT_IS_DISABLED; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; + extern const int INVALID_PARTITION_ID; } @@ -927,6 +928,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; + bool invalid_partition_id = false; String part_path = fs::path(relative_data_path) / part_name; String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; @@ -951,6 +953,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (isNotEnoughMemoryErrorCode(e.code())) throw; + if (e.code() == ErrorCodes::INVALID_PARTITION_ID) + invalid_partition_id = true; + broken = true; tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -963,7 +968,17 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Ignore and possibly delete broken parts that can appear as a result of hard server restart. if (broken) { - if (part->info.level == 0) + if (invalid_partition_id) + { + /// Special case when calculated partition id differs from partition ID in part name. + /// It may be because we did not notice change in function used in partition key, or the way how we calculate hash. + /// Just detach part in this case. + LOG_ERROR(log, "Detaching broken part {}{} because it has invalid partition id. If it happened after update, it is likely because of backward incompability. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); + std::lock_guard loading_lock(mutex); + broken_parts_to_detach.push_back(part); + ++suspicious_broken_parts; + } + else if (part->info.level == 0) { /// It is impossible to restore level 0 parts. LOG_ERROR(log, "Considering to remove broken part {}{} because it's impossible to repair.", getFullPathOnDisk(part_disk_ptr), part_name); From 2936fdd16c9b25a0caf888b31b8fdbc9ae89e592 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 10 Jun 2021 15:04:36 +0300 Subject: [PATCH 089/491] Always detach parts with wrong partition id. --- src/Common/ErrorCodes.cpp | 1 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +-- .../test_detach_part_wrong_partition_id.py | 32 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d840830bf28..428eafc7403 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -554,6 +554,7 @@ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ M(586, CANNOT_CREATE_FILE) \ + M(587, INVALID_PARTITION_ID) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 703cf32f743..690237e12d8 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -46,7 +46,7 @@ namespace ErrorCodes extern const int FILE_DOESNT_EXIST; extern const int NO_FILE_IN_DATA_PART; extern const int EXPECTED_END_OF_FILE; - extern const int CORRUPTED_DATA; + extern const int INVALID_PARTITION_ID; extern const int NOT_FOUND_EXPECTED_DATA_PART; extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; extern const int BAD_TTL_FILE; @@ -817,7 +817,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() throw Exception( "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id + " differs from partition ID in part name: " + info.partition_id, - ErrorCodes::CORRUPTED_DATA); + ErrorCodes::INVALID_PARTITION_ID); } void IMergeTreeDataPart::loadChecksums(bool require) diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py new file mode 100644 index 00000000000..5d41d5c394a --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -0,0 +1,32 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +# Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. +node1 = cluster.add_instance('node1', image='yandex/clickhouse-server', tag='21.6.3.14', stay_alive=True, with_installed_binary=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + +def test_detach_part_wrong_partition_id(start_cluster): + + # Here we create table with partition by UUID. + node1.query("create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()") + node1.query("insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)") + + # After restart, partition id will be different. + # There is a single 0-level part, which will become broken. + # We expect that it will not be removed (as usual for 0-level broken parts), + # but moved to /detached + node1.restart_with_latest_version() + + num_detached = node1.query("select count() from system.detached_parts") + assert num_detached == '1\n' From 3bfefa50e264b21dfa1b4b700a15c593e91da2c2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 10 Jun 2021 15:15:07 +0300 Subject: [PATCH 090/491] Add user arg to exec_in_container with cli --- tests/integration/helpers/cluster.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 82be7c6f678..a622f042956 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -247,7 +247,7 @@ class ClickHouseCluster: self.with_minio = False self.minio_dir = os.path.join(self.instances_dir, "minio") - self.minio_certs_dir = None # source for certificates + self.minio_certs_dir = None # source for certificates self.minio_host = "minio1" self.minio_ip = None self.minio_bucket = "root" @@ -327,7 +327,7 @@ class ClickHouseCluster: # available when with_mysql_client == True self.mysql_client_host = "mysql_client" self.mysql_client_container = None - + # available when with_mysql == True self.mysql_host = "mysql57" self.mysql_port = 3306 @@ -524,7 +524,7 @@ class ClickHouseCluster: return self.base_mysql8_cmd - def setup_mysql_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_mysql_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql_cluster = True env_variables['MYSQL_CLUSTER_PORT'] = str(self.mysql_port) env_variables['MYSQL_CLUSTER_ROOT_HOST'] = '%' @@ -548,7 +548,7 @@ class ClickHouseCluster: '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')] return self.base_postgres_cmd - def setup_postgres_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_postgres_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_postgres_cluster = True env_variables['POSTGRES_PORT'] = str(self.postgres_port) env_variables['POSTGRES2_DIR'] = self.postgres2_logs_dir @@ -644,7 +644,7 @@ class ClickHouseCluster: return self.base_mongo_cmd def setup_minio_cmd(self, instance, env_variables, docker_compose_yml_dir): - self.with_minio = True + self.with_minio = True cert_d = p.join(self.minio_dir, "certs") env_variables['MINIO_CERTS_DIR'] = cert_d env_variables['MINIO_PORT'] = str(self.minio_port) @@ -665,7 +665,7 @@ class ClickHouseCluster: def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None, macros=None, with_zookeeper=False, with_zookeeper_secure=False, - with_mysql_client=False, with_mysql=False, with_mysql8=False, with_mysql_cluster=False, + with_mysql_client=False, with_mysql=False, with_mysql8=False, with_mysql_cluster=False, with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, with_hdfs=False, with_kerberized_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, @@ -821,7 +821,7 @@ class ClickHouseCluster: if self.minio_certs_dir is None: self.minio_certs_dir = minio_certs_dir else: - raise Exception("Overwriting minio certs dir") + raise Exception("Overwriting minio certs dir") if with_cassandra and not self.with_cassandra: cmds.append(self.setup_cassandra_cmd(instance, env_variables, docker_compose_yml_dir)) @@ -889,7 +889,10 @@ class ClickHouseCluster: def exec_in_container(self, container_id, cmd, detach=False, nothrow=False, use_cli=True, **kwargs): if use_cli: logging.debug(f"run container_id:{container_id} detach:{detach} nothrow:{nothrow} cmd: {cmd}") - result = subprocess_check_call(["docker", "exec", container_id] + cmd, detach=detach, nothrow=nothrow) + exec_cmd = ["docker", "exec"] + if 'user' in kwargs: + exec_cmd += ['-u', kwargs['user']] + result = subprocess_check_call(exec_cmd + [container_id] + cmd, detach=detach, nothrow=nothrow) return result else: exec_id = self.docker_client.api.exec_create(container_id, cmd, **kwargs) @@ -1107,7 +1110,7 @@ class ClickHouseCluster: proxy_port=self.hdfs_kerberized_name_port, data_port=self.hdfs_kerberized_data_port, hdfs_ip=hdfs_ip, - kdc_ip=kdc_ip) + kdc_ip=kdc_ip) else: logging.debug("Create HDFSApi host={}".format("localhost")) hdfs_api = HDFSApi(user="root", host="localhost", data_port=self.hdfs_data_port, proxy_port=self.hdfs_name_port) @@ -1209,7 +1212,7 @@ class ClickHouseCluster: raise Exception("Can't wait Schema Registry to start") - + def wait_cassandra_to_start(self, timeout=180): self.cassandra_ip = self.get_instance_ip(self.cassandra_host) cass_client = cassandra.cluster.Cluster([self.cassandra_ip], port=self.cassandra_port, load_balancing_policy=RoundRobinPolicy()) @@ -1284,7 +1287,7 @@ class ClickHouseCluster: for dir in self.zookeeper_dirs_to_create: os.makedirs(dir) - + if self.use_keeper: # TODO: remove hardcoded paths from here for i in range(1,4): shutil.copy(os.path.join(HELPERS_DIR, f'keeper_config{i}.xml'), os.path.join(self.keeper_instance_dir_prefix + f"{i}", "config" )) @@ -2131,7 +2134,7 @@ class ClickHouseInstance: depends_on.append("postgres2") depends_on.append("postgres3") depends_on.append("postgres4") - + if self.with_kafka: depends_on.append("kafka1") depends_on.append("schema-registry") From aa90309abf07361349329dc769ab9748fbf09d82 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 10 Jun 2021 15:46:03 +0300 Subject: [PATCH 091/491] Never remove parts on start at all. Always move. --- src/Common/ErrorCodes.cpp | 1 - src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +- src/Storages/MergeTree/MergeTreeData.cpp | 73 ++----------------- 3 files changed, 10 insertions(+), 68 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 428eafc7403..d840830bf28 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -554,7 +554,6 @@ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ M(586, CANNOT_CREATE_FILE) \ - M(587, INVALID_PARTITION_ID) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 690237e12d8..703cf32f743 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -46,7 +46,7 @@ namespace ErrorCodes extern const int FILE_DOESNT_EXIST; extern const int NO_FILE_IN_DATA_PART; extern const int EXPECTED_END_OF_FILE; - extern const int INVALID_PARTITION_ID; + extern const int CORRUPTED_DATA; extern const int NOT_FOUND_EXPECTED_DATA_PART; extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; extern const int BAD_TTL_FILE; @@ -817,7 +817,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() throw Exception( "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id + " differs from partition ID in part name: " + info.partition_id, - ErrorCodes::INVALID_PARTITION_ID); + ErrorCodes::CORRUPTED_DATA); } void IMergeTreeDataPart::loadChecksums(bool require) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b3ec4d2666d..7d8838de0b9 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -123,7 +123,6 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int SUPPORT_IS_DISABLED; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; - extern const int INVALID_PARTITION_ID; } @@ -905,7 +904,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) std::mutex mutex; - DataPartsVector broken_parts_to_remove; DataPartsVector broken_parts_to_detach; size_t suspicious_broken_parts = 0; @@ -928,7 +926,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); auto part = createPart(part_name, part_info, single_disk_volume, part_name); bool broken = false; - bool invalid_partition_id = false; String part_path = fs::path(relative_data_path) / part_name; String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; @@ -953,9 +950,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (isNotEnoughMemoryErrorCode(e.code())) throw; - if (e.code() == ErrorCodes::INVALID_PARTITION_ID) - invalid_partition_id = true; - broken = true; tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -968,62 +962,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Ignore and possibly delete broken parts that can appear as a result of hard server restart. if (broken) { - if (invalid_partition_id) - { - /// Special case when calculated partition id differs from partition ID in part name. - /// It may be because we did not notice change in function used in partition key, or the way how we calculate hash. - /// Just detach part in this case. - LOG_ERROR(log, "Detaching broken part {}{} because it has invalid partition id. If it happened after update, it is likely because of backward incompability. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); - std::lock_guard loading_lock(mutex); - broken_parts_to_detach.push_back(part); - ++suspicious_broken_parts; - } - else if (part->info.level == 0) - { - /// It is impossible to restore level 0 parts. - LOG_ERROR(log, "Considering to remove broken part {}{} because it's impossible to repair.", getFullPathOnDisk(part_disk_ptr), part_name); - std::lock_guard loading_lock(mutex); - broken_parts_to_remove.push_back(part); - } - else - { - /// Count the number of parts covered by the broken part. If it is at least two, assume that - /// the broken part was created as a result of merging them and we won't lose data if we - /// delete it. - size_t contained_parts = 0; - - LOG_ERROR(log, "Part {}{} is broken. Looking for parts to replace it.", getFullPathOnDisk(part_disk_ptr), part_name); - - for (const auto & [contained_name, contained_disk_ptr] : part_names_with_disks) - { - if (contained_name == part_name) - continue; - - MergeTreePartInfo contained_part_info; - if (!MergeTreePartInfo::tryParsePartName(contained_name, &contained_part_info, format_version)) - continue; - - if (part->info.contains(contained_part_info)) - { - LOG_ERROR(log, "Found part {}{}", getFullPathOnDisk(contained_disk_ptr), contained_name); - ++contained_parts; - } - } - - if (contained_parts >= 2) - { - LOG_ERROR(log, "Considering to remove broken part {}{} because it covers at least 2 other parts", getFullPathOnDisk(part_disk_ptr), part_name); - std::lock_guard loading_lock(mutex); - broken_parts_to_remove.push_back(part); - } - else - { - LOG_ERROR(log, "Detaching broken part {}{} because it covers less than 2 parts. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); - std::lock_guard loading_lock(mutex); - broken_parts_to_detach.push_back(part); - ++suspicious_broken_parts; - } - } + /// Special case when calculated partition id differs from partition ID in part name. + /// It may be because we did not notice change in function used in partition key, or the way how we calculate hash. + /// Just detach part in this case. + LOG_ERROR(log, "Detaching broken part {}{}. If it happened after update, it is likely because of backward incompability. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); + std::lock_guard loading_lock(mutex); + broken_parts_to_detach.push_back(part); + ++suspicious_broken_parts; return; } @@ -1070,10 +1015,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) throw Exception("Suspiciously many (" + toString(suspicious_broken_parts) + ") broken parts to remove.", ErrorCodes::TOO_MANY_UNEXPECTED_DATA_PARTS); - for (auto & part : broken_parts_to_remove) - part->remove(); for (auto & part : broken_parts_to_detach) - part->renameToDetached(""); + part->renameToDetached("broken_on_start"); /// Delete from the set of current parts those parts that are covered by another part (those parts that From 6222fcffe008c83fb003df2bb5f88b2eadf4480e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 10 Jun 2021 15:47:55 +0300 Subject: [PATCH 092/491] Never remove parts on start at all. Always move. --- src/Storages/MergeTree/MergeTreeData.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7d8838de0b9..ed068325f45 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -959,12 +959,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) tryLogCurrentException(__PRETTY_FUNCTION__); } - /// Ignore and possibly delete broken parts that can appear as a result of hard server restart. + /// Ignore broken parts that can appear as a result of hard server restart. if (broken) { - /// Special case when calculated partition id differs from partition ID in part name. - /// It may be because we did not notice change in function used in partition key, or the way how we calculate hash. - /// Just detach part in this case. LOG_ERROR(log, "Detaching broken part {}{}. If it happened after update, it is likely because of backward incompability. You need to resolve this manually", getFullPathOnDisk(part_disk_ptr), part_name); std::lock_guard loading_lock(mutex); broken_parts_to_detach.push_back(part); From 8cbd9ec733453363a7a5146f661fd587dedd8f49 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 9 Jun 2021 07:46:49 +0000 Subject: [PATCH 093/491] Fixes --- .gitmodules | 6 +++--- contrib/libpqxx | 2 +- src/Storages/StoragePostgreSQL.cpp | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index ab7c8a7c94d..be44e3268e3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -210,9 +210,6 @@ [submodule "contrib/fast_float"] path = contrib/fast_float url = https://github.com/fastfloat/fast_float -[submodule "contrib/libpqxx"] - path = contrib/libpqxx - url = https://github.com/jtv/libpqxx [submodule "contrib/libpq"] path = contrib/libpq url = https://github.com/ClickHouse-Extras/libpq @@ -231,3 +228,6 @@ [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp url = https://github.com/ClickHouse-Extras/yaml-cpp.git +[submodule "contrib/libpqxx"] + path = contrib/libpqxx + url = https://github.com/ClickHouse-Extras/libpqxx.git diff --git a/contrib/libpqxx b/contrib/libpqxx index 8fcd332202e..1e29e01d794 160000 --- a/contrib/libpqxx +++ b/contrib/libpqxx @@ -1 +1 @@ -Subproject commit 8fcd332202e806d3d82ca27c60c8f7ebb6282aed +Subproject commit 1e29e01d7943fa1481e3694050e03bdddfa38703 diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 11f81e2f910..b1842c6b79f 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -267,11 +267,15 @@ private: struct StreamTo { pqxx::work tx; + Names columns; pqxx::stream_to stream; - StreamTo(pqxx::connection & connection, pqxx::table_path table_path, Names columns) + StreamTo(pqxx::connection & connection, pqxx::table_path table_, Names columns_) : tx(connection) - , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_path), connection.quote_columns(columns))) {} + , columns(std::move(columns_)) + , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_), connection.quote_columns(columns))) + { + } void complete() { From 296d18ed2a7005838a1264232bcff6c6e8343f23 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Thu, 10 Jun 2021 20:22:19 +0300 Subject: [PATCH 094/491] Fix the type matching tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил таблицы соответствия типов. --- docs/en/interfaces/formats.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index b36e9637179..e520d20e86e 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1251,11 +1251,11 @@ The table below shows supported data types and how they match ClickHouse [data t | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | -Elements of value `Array` type can be array or `Nullable` value. +Arrays can be nested and can have a value of the `Nullable` type as an argument. ClickHouse supports configurable precision of `Decimal` type. The `INSERT` query treats the Parquet `DECIMAL` type as the ClickHouse `Decimal128` type. -Unsupported Parquet data types: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +Unsupported Parquet data types: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. Data types of ClickHouse table columns can differ from the corresponding fields of the Parquet data inserted. When inserting data, ClickHouse interprets data types according to the table above and then [cast](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) the data to that data type which is set for the ClickHouse table column. @@ -1300,11 +1300,11 @@ The table below shows supported data types and how they match ClickHouse [data t | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `UTF8` | -| `STRING` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | -| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `-` | +| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | +| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | -Elements of value `Array` type can be array or value of the `Nullable` type. +Arrays can be nested and can have a value of the `Nullable` type as an argument. ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the Arrow `DECIMAL` type as the ClickHouse `Decimal128` type. @@ -1358,7 +1358,7 @@ The table below shows supported data types and how they match ClickHouse [data t | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | -Elements of value `Array` type can be array or value of the `Nullable` type. +Arrays can be nested and can have a value of the `Nullable` type as an argument. ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the ORC `DECIMAL` type as the ClickHouse `Decimal128` type. From 2613aee85661fb62737fe823873fbc1392eeaab8 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 11 Jun 2021 01:56:32 +0300 Subject: [PATCH 095/491] Some changes --- docs/en/sql-reference/data-types/array.md | 23 ++++++----- docs/en/sql-reference/data-types/nullable.md | 42 ++++++++++---------- docs/en/sql-reference/data-types/tuple.md | 2 + 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index ced7e399f45..ac9cbdd9a6e 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -74,25 +74,28 @@ Received exception from server (version 1.1.54388): Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. ``` -It is possible to use `size0` subcolumns that can be read without reading the whole column: +## Array Size {#array-size} + +It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use size`n-1`, where `n` is the wanted dimension. + +Query: ```sql -CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO t_arr VALUES ([1]) ([]) ([1, 2, 3]) ([1, 2]); +CREATE TABLE t_arr (`arr` Array(Array(Array(UInt32)))) ENGINE = MergeTree ORDER BY tuple(); -SELECT a.size0 FROM t_arr; +insert into t_arr values ([[[12, 13, 0, 1],[12]]]); + +SELECT arr.size0, arr.size1, arr.size2 FROM t_arr; ``` Result: ``` text -┌─a.size0─┐ -│ 1 │ -│ 0 │ -│ 3 │ -│ 2 │ -└─────────┘ +┌─arr.size0─┬─arr.size1─┬─arr.size2─┐ +│ 1 │ [2] │ [[4,1]] │ +└───────────┴───────────┴───────────┘ ``` [Original article](https://clickhouse.tech/docs/en/data_types/array/) diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 247129a0cf3..a526a99d9a8 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -20,6 +20,27 @@ To store `Nullable` type values in a table column, ClickHouse uses a separate fi !!! info "Note" Using `Nullable` almost always negatively affects performance, keep this in mind when designing your databases. +It is possible to find `NULL` values in a column by using `null` subcolumn without reading the whole column. + +``` sql +CREATE TABLE nullable ( `n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); + +SELECT n.null FROM nullable; +``` + +Result: + +``` text +┌─n.null─┐ +│ 0 │ +│ 1 │ +│ 0 │ +│ 1 │ +└────────┘ +``` + ## Usage Example {#usage-example} ``` sql @@ -41,25 +62,4 @@ SELECT x + y FROM t_null └────────────┘ ``` -It is possible to use `null` subcolumns that can be read without reading the whole column: - -``` sql -CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); - -INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); - -SELECT n.null FROM nullable; -``` - -Result: - -``` text -┌─n.null─┐ -│ 0 │ -│ 1 │ -│ 0 │ -│ 1 │ -└────────┘ -``` - [Original article](https://clickhouse.tech/docs/en/data_types/nullable/) diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index 21b4cc7e0d8..9092643ed42 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -47,6 +47,8 @@ SELECT tuple(1, NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` +## Addressing Tuple Element {#addressing-tuple-element} + It is possible to read elements of named tuples using indexes and names: ``` sql From 9cb9aca6b84bfd43049facc0b38d35541f6b73a1 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 11 Jun 2021 09:44:36 +0300 Subject: [PATCH 096/491] Update StoragePostgreSQL.cpp --- src/Storages/StoragePostgreSQL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index b1842c6b79f..ff2e12d8be3 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -151,7 +151,7 @@ public: } } - inserter->stream.write_values(row); + inserter->stream.write_values(row); // NOLINT } } @@ -273,7 +273,7 @@ private: StreamTo(pqxx::connection & connection, pqxx::table_path table_, Names columns_) : tx(connection) , columns(std::move(columns_)) - , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_), connection.quote_columns(columns))) + , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_), connection.quote_columns(columns))) // NOLINT { } From 79c69ea316c8180a220fa5a91fb8004e903d8c66 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 11 Jun 2021 11:29:39 +0300 Subject: [PATCH 097/491] Fix build. --- src/Storages/MergeTree/MergeTreeData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index ed068325f45..ccf625eb627 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -912,12 +912,12 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) ThreadPool pool(num_threads); - for (size_t i = 0; i < part_names_with_disks.size(); ++i) + for (auto & part_names_with_disk : part_names_with_disks) { - pool.scheduleOrThrowOnError([&, i] + pool.scheduleOrThrowOnError([&] { - const auto & part_name = part_names_with_disks[i].first; - const auto part_disk_ptr = part_names_with_disks[i].second; + const auto & part_name = part_names_with_disk.first; + const auto part_disk_ptr = part_names_with_disk.second; MergeTreePartInfo part_info; if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) From 6c9f701636abdd47b1401d398a19ec15dd4fd46c Mon Sep 17 00:00:00 2001 From: Storozhuk Kostiantyn <56565543+sand6255@users.noreply.github.com> Date: Fri, 11 Jun 2021 11:43:36 +0300 Subject: [PATCH 098/491] Wip my sql column comments support (#1) Implemented MySQL engine column comments support --- .../MySQL/InterpretersMySQLDDLQuery.cpp | 50 +++++++++++++++---- .../MySQL/tests/gtest_create_rewritten.cpp | 18 +++++-- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index cd2774de94a..a2a2111e153 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -67,10 +67,10 @@ static inline String resolveDatabase( return current_database != replica_clickhouse_database ? "" : replica_clickhouse_database; } -static inline NamesAndTypesList getColumnsList(ASTExpressionList * columns_define) +static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_definition) { NamesAndTypesList columns_name_and_type; - for (const auto & declare_column_ast : columns_define->children) + for (const auto & declare_column_ast : columns_definition->children) { const auto & declare_column = declare_column_ast->as(); @@ -117,6 +117,37 @@ static inline NamesAndTypesList getColumnsList(ASTExpressionList * columns_defin return columns_name_and_type; } +static ColumnsDescription getColumnsDescription(const NamesAndTypesList & columns_name_and_type, const ASTExpressionList * columns_definition) +{ + if (columns_name_and_type.size() != columns_definition->children.size()) + throw Exception("Columns of different size provided.", ErrorCodes::BAD_ARGUMENTS); + + ColumnsDescription columns_description; + ColumnDescription column_description; + + for ( + auto [column_name_and_type, declare_column_ast] = std::tuple{columns_name_and_type.begin(), columns_definition->children.begin()}; + column_name_and_type != columns_name_and_type.end(); + column_name_and_type++, + declare_column_ast++ + ) + { + const auto & declare_column = (*declare_column_ast)->as(); + String comment; + if (declare_column->column_options) + if (const auto * options = declare_column->column_options->as()) + if (options->changes.count("comment")) + comment = options->changes.at("comment")->as()->value.safeGet(); + + column_description.name = column_name_and_type->name; + column_description.type = column_name_and_type->type; + column_description.comment = std::move(comment); + columns_description.add(column_description); + } + + return columns_description; +} + static NamesAndTypesList getNames(const ASTFunction & expr, ContextPtr context, const NamesAndTypesList & columns) { if (expr.arguments->children.empty()) @@ -157,8 +188,8 @@ static NamesAndTypesList modifyPrimaryKeysToNonNullable(const NamesAndTypesList return non_nullable_primary_keys; } -static inline std::tuple getKeys( - ASTExpressionList * columns_define, ASTExpressionList * indices_define, ContextPtr context, NamesAndTypesList & columns) +static std::tuple getKeys( + ASTExpressionList * columns_definition, ASTExpressionList * indices_define, ContextPtr context, NamesAndTypesList & columns) { NameSet increment_columns; auto keys = makeASTFunction("tuple"); @@ -211,7 +242,7 @@ static inline std::tuplechildren) + for (const auto & declare_column_ast : columns_definition->children) { const auto & declare_column = declare_column_ast->as(); @@ -393,6 +424,7 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( NamesAndTypesList columns_name_and_type = getColumnsList(create_defines->columns); const auto & [primary_keys, unique_keys, keys, increment_columns] = getKeys(create_defines->columns, create_defines->indices, context, columns_name_and_type); + ColumnsDescription columns_description = getColumnsDescription(columns_name_and_type, create_defines->columns); if (primary_keys.empty()) throw Exception("The " + backQuoteIfNeed(mysql_database) + "." + backQuoteIfNeed(create_query.table) @@ -415,7 +447,7 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( /// Add _sign and _version columns. String sign_column_name = getUniqueColumnName(columns_name_and_type, "_sign"); String version_column_name = getUniqueColumnName(columns_name_and_type, "_version"); - columns->set(columns->columns, InterpreterCreateQuery::formatColumns(columns_name_and_type)); + columns->set(columns->columns, InterpreterCreateQuery::formatColumns(columns_description)); columns->columns->children.emplace_back(create_materialized_column_declaration(sign_column_name, "Int8", UInt64(1))); columns->columns->children.emplace_back(create_materialized_column_declaration(version_column_name, "UInt64", UInt64(1))); @@ -483,8 +515,8 @@ ASTs InterpreterRenameImpl::getRewrittenQueries( ASTRenameQuery::Elements elements; for (const auto & rename_element : rename_query.elements) { - const auto & to_database = resolveDatabase(rename_element.to.database, mysql_database, mapped_to_database, context); - const auto & from_database = resolveDatabase(rename_element.from.database, mysql_database, mapped_to_database, context); + const auto & to_database = resolveDatabase(rename_element.to.database, mysql_database, mapped_to_database, context); + const auto & from_database = resolveDatabase(rename_element.from.database, mysql_database, mapped_to_database, context); if ((from_database == mapped_to_database || to_database == mapped_to_database) && to_database != from_database) throw Exception("Cannot rename with other database for external ddl query.", ErrorCodes::NOT_IMPLEMENTED); @@ -647,7 +679,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries( } else if (alter_command->type == MySQLParser::ASTAlterCommand::RENAME_TABLE) { - const auto & to_database = resolveDatabase(alter_command->new_database_name, mysql_database, mapped_to_database, context); + const auto & to_database = resolveDatabase(alter_command->new_database_name, mysql_database, mapped_to_database, context); if (to_database != mapped_to_database) throw Exception("Cannot rename with other database for external ddl query.", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp index 77a14e780c5..37eede03e7f 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp @@ -61,7 +61,7 @@ TEST(MySQLCreateRewritten, ColumnsDataType) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' NOT NULL)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` " + mapped_type + + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` " + mapped_type + " COMMENT 'test_comment'" + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); @@ -75,7 +75,7 @@ TEST(MySQLCreateRewritten, ColumnsDataType) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' UNSIGNED)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" + + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(U" + mapped_type + ")" + " COMMENT 'test_comment'" + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); @@ -87,7 +87,7 @@ TEST(MySQLCreateRewritten, ColumnsDataType) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, test " + test_type + " COMMENT 'test_comment' UNSIGNED NOT NULL)", context_holder.context)), - "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` U" + mapped_type + + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` U" + mapped_type + " COMMENT 'test_comment'" + MATERIALIZEMYSQL_TABLE_COLUMNS + ") ENGINE = " "ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); } @@ -223,3 +223,15 @@ TEST(MySQLCreateRewritten, UniqueKeysConvert) std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(id, 18446744073709551) ORDER BY (code, name, tenant_id, id)"); } + +TEST(MySQLCreateRewritten, QueryWithColumnComments) +{ + tryRegisterFunctions(); + const auto & context_holder = getContext(); + + EXPECT_EQ(queryToString(tryRewrittenCreateQuery( + "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, `test` INT COMMENT 'test_comment')", context_holder.context)), + "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(Int32) COMMENT 'test_comment'" + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); +} From 52f60692c843c0a45c13539b56c4c1803e2400ec Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 11 Jun 2021 11:48:12 +0300 Subject: [PATCH 099/491] Do not optimize query plan for mutations. --- src/Interpreters/MutationsInterpreter.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index ca0b8257f6e..85b277d47ae 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -849,8 +849,11 @@ QueryPipelinePtr MutationsInterpreter::addStreamsForLaterStages(const std::vecto } } + QueryPlanOptimizationSettings do_not_optimize_plan; + do_not_optimize_plan.optimize_plan = false; + auto pipeline = plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(context), + do_not_optimize_plan, BuildQueryPipelineSettings::fromContext(context)); pipeline->addSimpleTransform([&](const Block & header) From 3dc718ff360c6894d6c613732bb51df4224d3975 Mon Sep 17 00:00:00 2001 From: Kostiantyn Storozhuk Date: Fri, 11 Jun 2021 19:29:12 +0800 Subject: [PATCH 100/491] style fixes --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 7 ++++--- src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index a2a2111e153..da741932a6c 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -33,8 +33,9 @@ namespace ErrorCodes { extern const int UNKNOWN_TYPE; extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; + extern const int NOT_IsMPLEMENTED; extern const int EMPTY_LIST_OF_COLUMNS_PASSED; + extern const int BAD_ARGUMENTS; } namespace MySQLInterpreter @@ -124,7 +125,7 @@ static ColumnsDescription getColumnsDescription(const NamesAndTypesList & column ColumnsDescription columns_description; ColumnDescription column_description; - + for ( auto [column_name_and_type, declare_column_ast] = std::tuple{columns_name_and_type.begin(), columns_definition->children.begin()}; column_name_and_type != columns_name_and_type.end(); @@ -138,7 +139,7 @@ static ColumnsDescription getColumnsDescription(const NamesAndTypesList & column if (const auto * options = declare_column->column_options->as()) if (options->changes.count("comment")) comment = options->changes.at("comment")->as()->value.safeGet(); - + column_description.name = column_name_and_type->name; column_description.type = column_name_and_type->type; column_description.comment = std::move(comment); diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp index 37eede03e7f..4bd65ae45a1 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp @@ -232,6 +232,6 @@ TEST(MySQLCreateRewritten, QueryWithColumnComments) EXPECT_EQ(queryToString(tryRewrittenCreateQuery( "CREATE TABLE `test_database`.`test_table_1`(`key` INT NOT NULL PRIMARY KEY, `test` INT COMMENT 'test_comment')", context_holder.context)), "CREATE TABLE test_database.test_table_1 (`key` Int32, `test` Nullable(Int32) COMMENT 'test_comment'" + - std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + + std::string(MATERIALIZEMYSQL_TABLE_COLUMNS) + ") ENGINE = ReplacingMergeTree(_version) PARTITION BY intDiv(key, 4294967) ORDER BY tuple(key)"); } From b2ddc98958cc464427d81b16333db92db27b8d35 Mon Sep 17 00:00:00 2001 From: Storozhuk Kostiantyn <56565543+sand6255@users.noreply.github.com> Date: Fri, 11 Jun 2021 14:34:03 +0300 Subject: [PATCH 101/491] Update InterpretersMySQLDDLQuery.cpp --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index da741932a6c..dd5dd311471 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -33,7 +33,7 @@ namespace ErrorCodes { extern const int UNKNOWN_TYPE; extern const int LOGICAL_ERROR; - extern const int NOT_IsMPLEMENTED; + extern const int NOT_IMPLEMENTED; extern const int EMPTY_LIST_OF_COLUMNS_PASSED; extern const int BAD_ARGUMENTS; } From cbbeb4001b2bb968debf06a97ce9c9eca8bf43a8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 11 Jun 2021 15:00:40 +0300 Subject: [PATCH 102/491] more changes --- tests/integration/helpers/cluster.py | 8 +++----- tests/integration/helpers/test_tools.py | 7 ++++--- .../test_backup_with_other_granularity/test.py | 13 +++++-------- .../test_cluster_copier/test_three_nodes.py | 10 +++++----- .../test_cluster_copier/test_two_nodes.py | 4 ++-- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 2aaa4e91cee..cc5c428ed2e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -575,7 +575,7 @@ class ClickHouseCluster: self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]) self.base_hdfs_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] - print("HDFS BASE CMD:{}".format(self.base_hdfs_cmd)) + logging.debug("HDFS BASE CMD:{self.base_hdfs_cmd)}") return self.base_hdfs_cmd def setup_kerberized_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir): @@ -1816,8 +1816,7 @@ class ClickHouseInstance: self.start_clickhouse(stop_start_wait_sec) def exec_in_container(self, cmd, detach=False, nothrow=False, **kwargs): - container_id = self.get_docker_handle().id - return self.cluster.exec_in_container(container_id, cmd, detach, nothrow, **kwargs) + return self.cluster.exec_in_container(self.docker_id, cmd, detach, nothrow, **kwargs) def contains_in_log(self, substring): result = self.exec_in_container( @@ -1857,8 +1856,7 @@ class ClickHouseInstance: ["bash", "-c", "echo $(if [ -e '{}' ]; then echo 'yes'; else echo 'no'; fi)".format(path)]) == 'yes\n' def copy_file_to_container(self, local_path, dest_path): - container_id = self.get_docker_handle().id - return self.cluster.copy_file_to_container(container_id, local_path, dest_path) + return self.cluster.copy_file_to_container(self.docker_id, local_path, dest_path) def get_process_pid(self, process_name): output = self.exec_in_container(["bash", "-c", diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 93478c4dd49..ef530c4836b 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -1,5 +1,6 @@ import difflib import time +import logging from io import IOBase @@ -56,7 +57,7 @@ def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_tim break time.sleep(sleep_time) except Exception as ex: - print(("assert_eq_with_retry retry {} exception {}".format(i + 1, ex))) + logging.exception(f"assert_eq_with_retry retry {i+1} exception {ex}") time.sleep(sleep_time) else: val = TSV(get_result(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, @@ -76,7 +77,7 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti break time.sleep(sleep_time) except Exception as ex: - print("contains_in_log_with_retry retry {} exception {}".format(i + 1, ex)) + logging.exception(f"contains_in_log_with_retry retry {i+1} exception {ex}") time.sleep(sleep_time) else: raise AssertionError("'{}' not found in logs".format(substring)) @@ -89,7 +90,7 @@ def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, setti break except Exception as ex: exception = ex - print("Failed to execute query '", query, "' on instance", instance.name, "will retry") + logging.exception(f"Failed to execute query '{query}' on instance '{instance.name}' will retry") time.sleep(sleep_time) else: raise exception diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 5ed1cb06787..832c1cf35ce 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -43,8 +43,8 @@ def test_backup_from_old_version(started_cluster): assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" - node1.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node1.exec_in_container(['find', '/var/lib/clickhouse/shadow/1/data/default/source_table']) + node1.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -81,8 +81,7 @@ def test_backup_from_old_version_setting(started_cluster): assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" - node2.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node2.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -123,8 +122,7 @@ def test_backup_from_old_version_config(started_cluster): assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" - node3.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node3.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -156,8 +154,7 @@ def test_backup_and_alter(started_cluster): node4.query("ALTER TABLE test.backup_table DROP PARTITION tuple()") - node4.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/ /var/lib/clickhouse/data/test/backup_table/detached']) + node4.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/', '/var/lib/clickhouse/data/test/backup_table/detached']) node4.query("ALTER TABLE test.backup_table ATTACH PARTITION tuple()") diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index acdc191154c..e9fcad7247f 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -39,7 +39,7 @@ class Task: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_taxi_data.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + logging.debug(f"Copied task file to container of '{instance_name}' instance. Path {self.container_task_file}") def start(self): @@ -187,10 +187,10 @@ def execute_task(started_cluster, task, cmd_options): task.start() zk = started_cluster.get_kazoo_client('zoo1') - print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) + logging.debug("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node - docker_api = docker.from_env().api + docker_api = started_cluster.docker_client.api copiers_exec_ids = [] cmd = ['/usr/bin/clickhouse', 'copier', @@ -201,9 +201,9 @@ def execute_task(started_cluster, task, cmd_options): '--base-dir', '/var/log/clickhouse-server/copier'] cmd += cmd_options - print(cmd) + logging.debug(f"execute_task cmd: {cmd}") - for instance_name, instance in started_cluster.instances.items(): + for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_three_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py index a6b2c82e00f..817c3571833 100644 --- a/tests/integration/test_cluster_copier/test_two_nodes.py +++ b/tests/integration/test_cluster_copier/test_two_nodes.py @@ -430,7 +430,7 @@ def execute_task(started_cluster, task, cmd_options): print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node - docker_api = docker.from_env().api + docker_api = started_cluster.docker_client.api copiers_exec_ids = [] cmd = ['/usr/bin/clickhouse', 'copier', @@ -443,7 +443,7 @@ def execute_task(started_cluster, task, cmd_options): print(cmd) - for instance_name, instance in started_cluster.instances.items(): + for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_two_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") From 73ff1728aeb72171550897149b2601f810432525 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 11 Jun 2021 15:41:48 +0300 Subject: [PATCH 103/491] rename flag to more generic name --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 16 ++++++++-------- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 2dfc3ff7a2d..8fe6a0a484b 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1236,7 +1236,7 @@ void IMergeTreeDataPart::remove() const } -void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3) const +void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_shared_data) const { String to = parent_to + "/" + relative_path; auto disk = volume->getDisk(); @@ -1248,7 +1248,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: checksums.txt is missing", fullPath(disk, to)); /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(to + "/", keep_shared_data); } else { @@ -1261,17 +1261,17 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->removeSharedFile(to + "/" + file, keep_s3); + disk->removeSharedFile(to + "/" + file, keep_shared_data); #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(to + "/" + file, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + disk->removeSharedFile(to + "/" + file, keep_shared_data); + disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_shared_data); + disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_shared_data); - disk->removeSharedRecursive(to, keep_s3); + disk->removeSharedRecursive(to, keep_shared_data); } catch (...) { @@ -1279,7 +1279,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_s3 LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(to + "/", keep_s3); + disk->removeSharedRecursive(to + "/", keep_shared_data); } } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index e05d0c5f487..f8ff7fe697a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -128,7 +128,7 @@ public: void remove() const; - void projectionRemove(const String & parent_to, bool keep_s3 = false) const; + void projectionRemove(const String & parent_to, bool keep_shared_data = false) const; /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. From 4773c793000c6b2c139b3947c09a26042dc8063b Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 11 Jun 2021 18:27:32 +0300 Subject: [PATCH 104/491] Fix ANTLR grammar --- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 2 +- src/Parsers/New/AST/ExplainQuery.cpp | 37 +- src/Parsers/New/AST/ExplainQuery.h | 13 +- src/Parsers/New/AST/TableExpr.cpp | 4 +- src/Parsers/New/AST/TableExpr.h | 4 +- src/Parsers/New/ClickHouseLexer.cpp | 2440 +++++----- src/Parsers/New/ClickHouseLexer.g4 | 1 + src/Parsers/New/ClickHouseLexer.h | 80 +- src/Parsers/New/ClickHouseParser.cpp | 4245 ++++++++--------- src/Parsers/New/ClickHouseParser.g4 | 25 +- src/Parsers/New/ClickHouseParser.h | 305 +- src/Parsers/New/ClickHouseParserVisitor.h | 4 +- src/Parsers/New/ParseTreeVisitor.h | 3 +- tests/queries/conftest.py | 16 +- tests/queries/query_test.py | 36 +- 15 files changed, 3488 insertions(+), 3727 deletions(-) diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index 1919e9d920c..dc02057f560 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -89,7 +89,7 @@ void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data) { if (identifier->isShort()) { - auto name = identifier->shortName(); + const auto & name = identifier->shortName(); auto subquery_it = data.subqueries.find(name); if (subquery_it != data.subqueries.end()) { diff --git a/src/Parsers/New/AST/ExplainQuery.cpp b/src/Parsers/New/AST/ExplainQuery.cpp index 154a401dde0..e6afd480f85 100644 --- a/src/Parsers/New/AST/ExplainQuery.cpp +++ b/src/Parsers/New/AST/ExplainQuery.cpp @@ -7,15 +7,37 @@ namespace DB::AST { -ExplainQuery::ExplainQuery(PtrTo query) : Query{query} +// static +PtrTo ExplainQuery::createExplainAST(PtrTo query) +{ + return PtrTo(new ExplainQuery(QueryType::AST, {query})); +} + +// static +PtrTo ExplainQuery::createExplainSyntax(PtrTo query) +{ + return PtrTo(new ExplainQuery(QueryType::SYNTAX, {query})); +} + +ExplainQuery::ExplainQuery(QueryType type, PtrList exprs) : Query{exprs}, query_type(type) { } ASTPtr ExplainQuery::convertToOld() const { - auto query = std::make_shared(ASTExplainQuery::AnalyzedSyntax); + ASTPtr query; - query->setExplainedQuery(get(QUERY)->convertToOld()); + switch (query_type) + { + case QueryType::AST: + query = std::make_shared(ASTExplainQuery::ParsedAST); + break; + case QueryType::SYNTAX: + query = std::make_shared(ASTExplainQuery::AnalyzedSyntax); + break; + } + + query->as()->setExplainedQuery(get(QUERY)->convertToOld()); return query; } @@ -27,9 +49,14 @@ namespace DB using namespace DB::AST; -antlrcpp::Any ParseTreeVisitor::visitExplainStmt(ClickHouseParser::ExplainStmtContext *ctx) +antlrcpp::Any ParseTreeVisitor::visitExplainASTStmt(ClickHouseParser::ExplainASTStmtContext *ctx) { - return std::make_shared(visit(ctx->query()).as>()); + return ExplainQuery::createExplainAST(visit(ctx->query()).as>()); +} + +antlrcpp::Any ParseTreeVisitor::visitExplainSyntaxStmt(ClickHouseParser::ExplainSyntaxStmtContext *ctx) +{ + return ExplainQuery::createExplainSyntax(visit(ctx->query()).as>()); } } diff --git a/src/Parsers/New/AST/ExplainQuery.h b/src/Parsers/New/AST/ExplainQuery.h index 6d6b42cca2d..53bc63e7fd5 100644 --- a/src/Parsers/New/AST/ExplainQuery.h +++ b/src/Parsers/New/AST/ExplainQuery.h @@ -9,7 +9,8 @@ namespace DB::AST class ExplainQuery : public Query { public: - explicit ExplainQuery(PtrTo query); + static PtrTo createExplainAST(PtrTo query); + static PtrTo createExplainSyntax(PtrTo query); ASTPtr convertToOld() const override; @@ -18,6 +19,16 @@ class ExplainQuery : public Query { QUERY = 0, // Query }; + + enum class QueryType + { + AST, + SYNTAX, + }; + + const QueryType query_type; + + ExplainQuery(QueryType type, PtrList exprs); }; } diff --git a/src/Parsers/New/AST/TableExpr.cpp b/src/Parsers/New/AST/TableExpr.cpp index 9d79a797085..63768e16f53 100644 --- a/src/Parsers/New/AST/TableExpr.cpp +++ b/src/Parsers/New/AST/TableExpr.cpp @@ -21,7 +21,7 @@ TableArgExpr::TableArgExpr(PtrTo function) : INode{function} { } -TableArgExpr::TableArgExpr(PtrTo identifier) : INode{identifier} +TableArgExpr::TableArgExpr(PtrTo identifier) : INode{identifier} { } @@ -149,7 +149,7 @@ antlrcpp::Any ParseTreeVisitor::visitTableArgExpr(ClickHouseParser::TableArgExpr { if (ctx->literal()) return std::make_shared(visit(ctx->literal()).as>()); if (ctx->tableFunctionExpr()) return std::make_shared(visit(ctx->tableFunctionExpr()).as>()); - if (ctx->tableIdentifier()) return std::make_shared(visit(ctx->tableIdentifier()).as>()); + if (ctx->identifier()) return std::make_shared(visit(ctx->identifier()).as>()); __builtin_unreachable(); } diff --git a/src/Parsers/New/AST/TableExpr.h b/src/Parsers/New/AST/TableExpr.h index 08a443fd217..1d893753023 100644 --- a/src/Parsers/New/AST/TableExpr.h +++ b/src/Parsers/New/AST/TableExpr.h @@ -11,14 +11,14 @@ class TableArgExpr : public INode public: explicit TableArgExpr(PtrTo literal); explicit TableArgExpr(PtrTo function); - explicit TableArgExpr(PtrTo identifier); + explicit TableArgExpr(PtrTo identifier); ASTPtr convertToOld() const override; private: enum ChildIndex : UInt8 { - EXPR = 0, // Literal or TableFunctionExpr or TableIdentifier + EXPR = 0, // Literal or TableFunctionExpr or Identifier }; }; diff --git a/src/Parsers/New/ClickHouseLexer.cpp b/src/Parsers/New/ClickHouseLexer.cpp index f5db3a71dee..7fb2a0effaa 100644 --- a/src/Parsers/New/ClickHouseLexer.cpp +++ b/src/Parsers/New/ClickHouseLexer.cpp @@ -62,21 +62,21 @@ std::vector ClickHouseLexer::_serializedATN; std::vector ClickHouseLexer::_ruleNames = { u8"ADD", u8"AFTER", u8"ALIAS", u8"ALL", u8"ALTER", u8"AND", u8"ANTI", - u8"ANY", u8"ARRAY", u8"AS", u8"ASCENDING", u8"ASOF", u8"ASYNC", u8"ATTACH", - u8"BETWEEN", u8"BOTH", u8"BY", u8"CASE", u8"CAST", u8"CHECK", u8"CLEAR", - u8"CLUSTER", u8"CODEC", u8"COLLATE", u8"COLUMN", u8"COMMENT", u8"CONSTRAINT", - u8"CREATE", u8"CROSS", u8"CUBE", u8"DATABASE", u8"DATABASES", u8"DATE", - u8"DAY", u8"DEDUPLICATE", u8"DEFAULT", u8"DELAY", u8"DELETE", u8"DESC", - u8"DESCENDING", u8"DESCRIBE", u8"DETACH", u8"DICTIONARIES", u8"DICTIONARY", - u8"DISK", u8"DISTINCT", u8"DISTRIBUTED", u8"DROP", u8"ELSE", u8"END", - u8"ENGINE", u8"EVENTS", u8"EXISTS", u8"EXPLAIN", u8"EXPRESSION", u8"EXTRACT", - u8"FETCHES", u8"FINAL", u8"FIRST", u8"FLUSH", u8"FOR", u8"FORMAT", u8"FREEZE", - u8"FROM", u8"FULL", u8"FUNCTION", u8"GLOBAL", u8"GRANULARITY", u8"GROUP", - u8"HAVING", u8"HIERARCHICAL", u8"HOUR", u8"ID", u8"IF", u8"ILIKE", u8"IN", - u8"INDEX", u8"INF", u8"INJECTIVE", u8"INNER", u8"INSERT", u8"INTERVAL", + u8"ANY", u8"ARRAY", u8"AS", u8"ASCENDING", u8"ASOF", u8"AST", u8"ASYNC", + u8"ATTACH", u8"BETWEEN", u8"BOTH", u8"BY", u8"CASE", u8"CAST", u8"CHECK", + u8"CLEAR", u8"CLUSTER", u8"CODEC", u8"COLLATE", u8"COLUMN", u8"COMMENT", + u8"CONSTRAINT", u8"CREATE", u8"CROSS", u8"CUBE", u8"DATABASE", u8"DATABASES", + u8"DATE", u8"DAY", u8"DEDUPLICATE", u8"DEFAULT", u8"DELAY", u8"DELETE", + u8"DESC", u8"DESCENDING", u8"DESCRIBE", u8"DETACH", u8"DICTIONARIES", + u8"DICTIONARY", u8"DISK", u8"DISTINCT", u8"DISTRIBUTED", u8"DROP", u8"ELSE", + u8"END", u8"ENGINE", u8"EVENTS", u8"EXISTS", u8"EXPLAIN", u8"EXPRESSION", + u8"EXTRACT", u8"FETCHES", u8"FINAL", u8"FIRST", u8"FLUSH", u8"FOR", u8"FORMAT", + u8"FREEZE", u8"FROM", u8"FULL", u8"FUNCTION", u8"GLOBAL", u8"GRANULARITY", + u8"GROUP", u8"HAVING", u8"HIERARCHICAL", u8"HOUR", u8"ID", u8"IF", u8"ILIKE", + u8"IN", u8"INDEX", u8"INF", u8"INJECTIVE", u8"INNER", u8"INSERT", u8"INTERVAL", u8"INTO", u8"IS", u8"IS_OBJECT_ID", u8"JOIN", u8"KEY", u8"KILL", u8"LAST", u8"LAYOUT", u8"LEADING", u8"LEFT", u8"LIFETIME", u8"LIKE", u8"LIMIT", - u8"LIVE", u8"LOCAL", u8"LOGS", u8"MATERIALIZED", u8"MATERIALIZE", u8"MAX", + u8"LIVE", u8"LOCAL", u8"LOGS", u8"MATERIALIZE", u8"MATERIALIZED", u8"MAX", u8"MERGES", u8"MIN", u8"MINUTE", u8"MODIFY", u8"MONTH", u8"MOVE", u8"MUTATION", u8"NAN_SQL", u8"NO", u8"NOT", u8"NULL_SQL", u8"NULLS", u8"OFFSET", u8"ON", u8"OPTIMIZE", u8"OR", u8"ORDER", u8"OUTER", u8"OUTFILE", u8"PARTITION", @@ -121,7 +121,7 @@ std::vector ClickHouseLexer::_literalNames = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", - "", u8"'false'", u8"'true'", "", "", "", "", "", "", u8"'->'", u8"'*'", + "", "", u8"'false'", u8"'true'", "", "", "", "", "", "", u8"'->'", u8"'*'", u8"'`'", u8"'\\'", u8"':'", u8"','", u8"'||'", u8"'-'", u8"'.'", u8"'=='", u8"'='", u8"'>='", u8"'>'", u8"'{'", u8"'['", u8"'<='", u8"'('", u8"'<'", "", u8"'%'", u8"'+'", u8"'?'", u8"'\"'", u8"'''", u8"'}'", u8"']'", u8"')'", @@ -130,21 +130,21 @@ std::vector ClickHouseLexer::_literalNames = { std::vector ClickHouseLexer::_symbolicNames = { "", u8"ADD", u8"AFTER", u8"ALIAS", u8"ALL", u8"ALTER", u8"AND", u8"ANTI", - u8"ANY", u8"ARRAY", u8"AS", u8"ASCENDING", u8"ASOF", u8"ASYNC", u8"ATTACH", - u8"BETWEEN", u8"BOTH", u8"BY", u8"CASE", u8"CAST", u8"CHECK", u8"CLEAR", - u8"CLUSTER", u8"CODEC", u8"COLLATE", u8"COLUMN", u8"COMMENT", u8"CONSTRAINT", - u8"CREATE", u8"CROSS", u8"CUBE", u8"DATABASE", u8"DATABASES", u8"DATE", - u8"DAY", u8"DEDUPLICATE", u8"DEFAULT", u8"DELAY", u8"DELETE", u8"DESC", - u8"DESCENDING", u8"DESCRIBE", u8"DETACH", u8"DICTIONARIES", u8"DICTIONARY", - u8"DISK", u8"DISTINCT", u8"DISTRIBUTED", u8"DROP", u8"ELSE", u8"END", - u8"ENGINE", u8"EVENTS", u8"EXISTS", u8"EXPLAIN", u8"EXPRESSION", u8"EXTRACT", - u8"FETCHES", u8"FINAL", u8"FIRST", u8"FLUSH", u8"FOR", u8"FORMAT", u8"FREEZE", - u8"FROM", u8"FULL", u8"FUNCTION", u8"GLOBAL", u8"GRANULARITY", u8"GROUP", - u8"HAVING", u8"HIERARCHICAL", u8"HOUR", u8"ID", u8"IF", u8"ILIKE", u8"IN", - u8"INDEX", u8"INF", u8"INJECTIVE", u8"INNER", u8"INSERT", u8"INTERVAL", + u8"ANY", u8"ARRAY", u8"AS", u8"ASCENDING", u8"ASOF", u8"AST", u8"ASYNC", + u8"ATTACH", u8"BETWEEN", u8"BOTH", u8"BY", u8"CASE", u8"CAST", u8"CHECK", + u8"CLEAR", u8"CLUSTER", u8"CODEC", u8"COLLATE", u8"COLUMN", u8"COMMENT", + u8"CONSTRAINT", u8"CREATE", u8"CROSS", u8"CUBE", u8"DATABASE", u8"DATABASES", + u8"DATE", u8"DAY", u8"DEDUPLICATE", u8"DEFAULT", u8"DELAY", u8"DELETE", + u8"DESC", u8"DESCENDING", u8"DESCRIBE", u8"DETACH", u8"DICTIONARIES", + u8"DICTIONARY", u8"DISK", u8"DISTINCT", u8"DISTRIBUTED", u8"DROP", u8"ELSE", + u8"END", u8"ENGINE", u8"EVENTS", u8"EXISTS", u8"EXPLAIN", u8"EXPRESSION", + u8"EXTRACT", u8"FETCHES", u8"FINAL", u8"FIRST", u8"FLUSH", u8"FOR", u8"FORMAT", + u8"FREEZE", u8"FROM", u8"FULL", u8"FUNCTION", u8"GLOBAL", u8"GRANULARITY", + u8"GROUP", u8"HAVING", u8"HIERARCHICAL", u8"HOUR", u8"ID", u8"IF", u8"ILIKE", + u8"IN", u8"INDEX", u8"INF", u8"INJECTIVE", u8"INNER", u8"INSERT", u8"INTERVAL", u8"INTO", u8"IS", u8"IS_OBJECT_ID", u8"JOIN", u8"KEY", u8"KILL", u8"LAST", u8"LAYOUT", u8"LEADING", u8"LEFT", u8"LIFETIME", u8"LIKE", u8"LIMIT", - u8"LIVE", u8"LOCAL", u8"LOGS", u8"MATERIALIZED", u8"MATERIALIZE", u8"MAX", + u8"LIVE", u8"LOCAL", u8"LOGS", u8"MATERIALIZE", u8"MATERIALIZED", u8"MAX", u8"MERGES", u8"MIN", u8"MINUTE", u8"MODIFY", u8"MONTH", u8"MOVE", u8"MUTATION", u8"NAN_SQL", u8"NO", u8"NOT", u8"NULL_SQL", u8"NULLS", u8"OFFSET", u8"ON", u8"OPTIMIZE", u8"OR", u8"ORDER", u8"OUTER", u8"OUTFILE", u8"PARTITION", @@ -188,7 +188,7 @@ ClickHouseLexer::Initializer::Initializer() { _serializedATN = { 0x3, 0x608b, 0xa72a, 0x8133, 0xb9ed, 0x417c, 0x3be7, 0x7786, 0x5964, - 0x2, 0xdf, 0x7fd, 0x8, 0x1, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, + 0x2, 0xe0, 0x803, 0x8, 0x1, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, 0x4, 0x4, 0x9, 0x4, 0x4, 0x5, 0x9, 0x5, 0x4, 0x6, 0x9, 0x6, 0x4, 0x7, 0x9, 0x7, 0x4, 0x8, 0x9, 0x8, 0x4, 0x9, 0x9, 0x9, 0x4, 0xa, 0x9, 0xa, 0x4, 0xb, 0x9, 0xb, 0x4, 0xc, 0x9, 0xc, 0x4, 0xd, 0x9, 0xd, 0x4, 0xe, @@ -265,291 +265,292 @@ ClickHouseLexer::Initializer::Initializer() { 0x4, 0xf2, 0x9, 0xf2, 0x4, 0xf3, 0x9, 0xf3, 0x4, 0xf4, 0x9, 0xf4, 0x4, 0xf5, 0x9, 0xf5, 0x4, 0xf6, 0x9, 0xf6, 0x4, 0xf7, 0x9, 0xf7, 0x4, 0xf8, 0x9, 0xf8, 0x4, 0xf9, 0x9, 0xf9, 0x4, 0xfa, 0x9, 0xfa, 0x4, 0xfb, 0x9, - 0xfb, 0x4, 0xfc, 0x9, 0xfc, 0x3, 0x2, 0x3, 0x2, 0x3, 0x2, 0x3, 0x2, - 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, - 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x5, 0x3, 0x5, - 0x3, 0x5, 0x3, 0x5, 0x3, 0x6, 0x3, 0x6, 0x3, 0x6, 0x3, 0x6, 0x3, 0x6, - 0x3, 0x6, 0x3, 0x7, 0x3, 0x7, 0x3, 0x7, 0x3, 0x7, 0x3, 0x8, 0x3, 0x8, - 0x3, 0x8, 0x3, 0x8, 0x3, 0x8, 0x3, 0x9, 0x3, 0x9, 0x3, 0x9, 0x3, 0x9, - 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xb, - 0x3, 0xb, 0x3, 0xb, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, + 0xfb, 0x4, 0xfc, 0x9, 0xfc, 0x4, 0xfd, 0x9, 0xfd, 0x3, 0x2, 0x3, 0x2, + 0x3, 0x2, 0x3, 0x2, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, + 0x3, 0x3, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, 0x3, 0x4, + 0x3, 0x5, 0x3, 0x5, 0x3, 0x5, 0x3, 0x5, 0x3, 0x6, 0x3, 0x6, 0x3, 0x6, + 0x3, 0x6, 0x3, 0x6, 0x3, 0x6, 0x3, 0x7, 0x3, 0x7, 0x3, 0x7, 0x3, 0x7, + 0x3, 0x8, 0x3, 0x8, 0x3, 0x8, 0x3, 0x8, 0x3, 0x8, 0x3, 0x9, 0x3, 0x9, + 0x3, 0x9, 0x3, 0x9, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, 0x3, 0xa, + 0x3, 0xa, 0x3, 0xb, 0x3, 0xb, 0x3, 0xb, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, - 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x238, 0xa, 0xc, 0x3, 0xd, 0x3, 0xd, 0x3, - 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, + 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x3, 0xc, 0x5, 0xc, 0x23a, 0xa, 0xc, 0x3, + 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xd, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xe, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, 0xf, 0x3, - 0xf, 0x3, 0xf, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, - 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x11, 0x3, 0x11, 0x3, 0x11, 0x3, - 0x11, 0x3, 0x11, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x13, 0x3, 0x13, - 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, - 0x14, 0x3, 0x14, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, + 0xf, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, 0x3, 0x10, + 0x3, 0x10, 0x3, 0x11, 0x3, 0x11, 0x3, 0x11, 0x3, 0x11, 0x3, 0x11, 0x3, + 0x11, 0x3, 0x11, 0x3, 0x11, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, 0x3, 0x12, + 0x3, 0x12, 0x3, 0x13, 0x3, 0x13, 0x3, 0x13, 0x3, 0x14, 0x3, 0x14, 0x3, + 0x14, 0x3, 0x14, 0x3, 0x14, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x15, 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, 0x3, 0x16, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, 0x3, 0x17, - 0x3, 0x17, 0x3, 0x17, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, + 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x18, 0x3, 0x19, 0x3, 0x19, 0x3, 0x19, 0x3, 0x19, 0x3, 0x19, - 0x3, 0x19, 0x3, 0x19, 0x3, 0x19, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, - 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, - 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1c, 0x3, - 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, - 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, - 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, + 0x3, 0x19, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, + 0x1a, 0x3, 0x1a, 0x3, 0x1a, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, + 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1b, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, + 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1c, 0x3, 0x1d, 0x3, 0x1d, + 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, + 0x1d, 0x3, 0x1d, 0x3, 0x1d, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1e, 0x3, 0x1f, 0x3, 0x1f, 0x3, 0x1f, 0x3, - 0x1f, 0x3, 0x1f, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, - 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x21, 0x3, 0x21, 0x3, - 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, - 0x3, 0x21, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, - 0x23, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, - 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, - 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, - 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x26, 0x3, 0x26, 0x3, + 0x1f, 0x3, 0x1f, 0x3, 0x1f, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, 0x3, 0x20, + 0x3, 0x20, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, + 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x21, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, + 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, 0x22, 0x3, + 0x22, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x3, 0x23, 0x3, 0x24, + 0x3, 0x24, 0x3, 0x24, 0x3, 0x24, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, + 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, 0x3, 0x25, + 0x3, 0x25, 0x3, 0x25, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x26, 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, - 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, 0x3, 0x28, 0x3, 0x28, 0x3, - 0x28, 0x3, 0x28, 0x3, 0x28, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, - 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, 0x3, - 0x29, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, - 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, - 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, - 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, - 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, - 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, - 0x2d, 0x3, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, - 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, - 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, - 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, - 0x30, 0x3, 0x30, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, - 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x33, 0x3, - 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, - 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, - 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, - 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x37, 0x3, 0x37, 0x3, - 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x38, - 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, - 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, - 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x3a, 0x3, - 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, + 0x3, 0x27, 0x3, 0x27, 0x3, 0x27, 0x3, 0x28, 0x3, 0x28, 0x3, 0x28, 0x3, + 0x28, 0x3, 0x28, 0x3, 0x28, 0x3, 0x28, 0x3, 0x29, 0x3, 0x29, 0x3, 0x29, + 0x3, 0x29, 0x3, 0x29, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, + 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, 0x3, 0x2a, + 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, + 0x2b, 0x3, 0x2b, 0x3, 0x2b, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, + 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, + 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, + 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, + 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, + 0x3, 0x2e, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, + 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, + 0x3, 0x30, 0x3, 0x30, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, + 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, + 0x3, 0x31, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, + 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x34, 0x3, 0x34, + 0x3, 0x34, 0x3, 0x34, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, + 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, + 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, + 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x37, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, + 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x39, 0x3, + 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, + 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, + 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3e, 0x3, - 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x3f, - 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, - 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, - 0x3, 0x41, 0x3, 0x41, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, - 0x42, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, - 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, - 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, - 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, - 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, + 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3f, 0x3, 0x3f, + 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, + 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, + 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, + 0x42, 0x3, 0x42, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, + 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, + 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, + 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, + 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x47, 0x3, 0x47, 0x3, 0x47, 0x3, 0x47, 0x3, - 0x47, 0x3, 0x47, 0x3, 0x47, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, - 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, - 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, - 0x3, 0x49, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, - 0x4b, 0x3, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x3, 0x4c, - 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4e, 0x3, 0x4e, 0x3, 0x4e, 0x3, + 0x47, 0x3, 0x47, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, + 0x3, 0x48, 0x3, 0x48, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, + 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, + 0x3, 0x49, 0x3, 0x49, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x3, + 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4c, 0x3, 0x4c, 0x3, 0x4c, + 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4e, 0x3, 0x4e, 0x3, 0x4e, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, - 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, - 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x5, 0x4f, 0x414, 0xa, 0x4f, 0x3, 0x50, - 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, - 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, - 0x3, 0x51, 0x3, 0x51, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, - 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, - 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x54, 0x3, - 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x55, 0x3, 0x55, 0x3, 0x55, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, - 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x58, 0x3, - 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x3, - 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, - 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, - 0x5c, 0x3, 0x5c, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, - 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, - 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, - 0x3, 0x5f, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, - 0x60, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x62, - 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x63, 0x3, - 0x63, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, - 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, - 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, + 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, + 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, + 0x3, 0x50, 0x3, 0x50, 0x5, 0x50, 0x41a, 0xa, 0x50, 0x3, 0x51, 0x3, 0x51, + 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, + 0x51, 0x3, 0x51, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, + 0x3, 0x52, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, 0x53, 0x3, + 0x53, 0x3, 0x53, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, + 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x55, 0x3, 0x55, 0x3, + 0x55, 0x3, 0x55, 0x3, 0x55, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x57, + 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, + 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x3, 0x58, + 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x59, 0x3, 0x59, 0x3, + 0x59, 0x3, 0x59, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, + 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5b, 0x3, 0x5c, 0x3, + 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5d, + 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, + 0x5d, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5f, + 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, + 0x5f, 0x3, 0x5f, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, + 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, + 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x3, 0x63, 0x3, 0x63, + 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x3, 0x64, 0x3, 0x64, 0x3, + 0x64, 0x3, 0x64, 0x3, 0x64, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, - 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, - 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, - 0x67, 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, 0x69, 0x3, 0x69, - 0x3, 0x69, 0x3, 0x69, 0x3, 0x69, 0x3, 0x69, 0x3, 0x69, 0x3, 0x6a, 0x3, - 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6b, - 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6c, 0x3, - 0x6c, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6d, - 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6d, 0x3, - 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6f, 0x3, 0x6f, 0x3, 0x6f, - 0x3, 0x70, 0x3, 0x70, 0x3, 0x70, 0x3, 0x70, 0x3, 0x71, 0x3, 0x71, 0x3, - 0x71, 0x3, 0x71, 0x3, 0x71, 0x3, 0x72, 0x3, 0x72, 0x3, 0x72, 0x3, 0x72, - 0x3, 0x72, 0x3, 0x72, 0x3, 0x73, 0x3, 0x73, 0x3, 0x73, 0x3, 0x73, 0x3, - 0x73, 0x3, 0x73, 0x3, 0x73, 0x3, 0x74, 0x3, 0x74, 0x3, 0x74, 0x3, 0x75, - 0x3, 0x75, 0x3, 0x75, 0x3, 0x75, 0x3, 0x75, 0x3, 0x75, 0x3, 0x75, 0x3, - 0x75, 0x3, 0x75, 0x3, 0x76, 0x3, 0x76, 0x3, 0x76, 0x3, 0x77, 0x3, 0x77, - 0x3, 0x77, 0x3, 0x77, 0x3, 0x77, 0x3, 0x77, 0x3, 0x78, 0x3, 0x78, 0x3, - 0x78, 0x3, 0x78, 0x3, 0x78, 0x3, 0x78, 0x3, 0x79, 0x3, 0x79, 0x3, 0x79, - 0x3, 0x79, 0x3, 0x79, 0x3, 0x79, 0x3, 0x79, 0x3, 0x79, 0x3, 0x7a, 0x3, - 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, - 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, - 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7c, 0x3, 0x7c, - 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, - 0x7c, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, - 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, + 0x65, 0x3, 0x65, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, + 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, + 0x66, 0x3, 0x66, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x3, 0x68, + 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, 0x68, 0x3, + 0x69, 0x3, 0x69, 0x3, 0x69, 0x3, 0x69, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, + 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6b, 0x3, 0x6b, 0x3, + 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6b, 0x3, 0x6c, 0x3, 0x6c, + 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6d, 0x3, 0x6d, 0x3, + 0x6d, 0x3, 0x6d, 0x3, 0x6d, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, + 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6f, 0x3, + 0x6f, 0x3, 0x6f, 0x3, 0x6f, 0x3, 0x70, 0x3, 0x70, 0x3, 0x70, 0x3, 0x71, + 0x3, 0x71, 0x3, 0x71, 0x3, 0x71, 0x3, 0x72, 0x3, 0x72, 0x3, 0x72, 0x3, + 0x72, 0x3, 0x72, 0x3, 0x73, 0x3, 0x73, 0x3, 0x73, 0x3, 0x73, 0x3, 0x73, + 0x3, 0x73, 0x3, 0x74, 0x3, 0x74, 0x3, 0x74, 0x3, 0x74, 0x3, 0x74, 0x3, + 0x74, 0x3, 0x74, 0x3, 0x75, 0x3, 0x75, 0x3, 0x75, 0x3, 0x76, 0x3, 0x76, + 0x3, 0x76, 0x3, 0x76, 0x3, 0x76, 0x3, 0x76, 0x3, 0x76, 0x3, 0x76, 0x3, + 0x76, 0x3, 0x77, 0x3, 0x77, 0x3, 0x77, 0x3, 0x78, 0x3, 0x78, 0x3, 0x78, + 0x3, 0x78, 0x3, 0x78, 0x3, 0x78, 0x3, 0x79, 0x3, 0x79, 0x3, 0x79, 0x3, + 0x79, 0x3, 0x79, 0x3, 0x79, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, + 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7a, 0x3, 0x7b, 0x3, 0x7b, 0x3, + 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, 0x3, 0x7b, + 0x3, 0x7b, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, + 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7c, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, + 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7d, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, 0x3, 0x7e, - 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, - 0x7f, 0x3, 0x7f, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, - 0x3, 0x80, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, - 0x81, 0x3, 0x81, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, - 0x3, 0x82, 0x3, 0x82, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, - 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, - 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, 0x85, 0x3, 0x85, 0x3, - 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x86, - 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, - 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, - 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, 0x88, 0x3, 0x88, 0x3, 0x88, 0x3, - 0x88, 0x3, 0x88, 0x3, 0x88, 0x3, 0x88, 0x3, 0x89, 0x3, 0x89, 0x3, 0x89, - 0x3, 0x89, 0x3, 0x89, 0x3, 0x89, 0x3, 0x89, 0x3, 0x8a, 0x3, 0x8a, 0x3, - 0x8a, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8b, 0x3, 0x8b, - 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8c, 0x3, - 0x8c, 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8d, 0x3, 0x8d, 0x3, 0x8d, - 0x3, 0x8d, 0x3, 0x8d, 0x3, 0x8d, 0x3, 0x8e, 0x3, 0x8e, 0x3, 0x8e, 0x3, - 0x8e, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, - 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x90, 0x3, 0x90, 0x3, 0x90, 0x3, - 0x90, 0x3, 0x90, 0x3, 0x91, 0x3, 0x91, 0x3, 0x91, 0x3, 0x91, 0x3, 0x91, - 0x3, 0x91, 0x3, 0x91, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, 0x3, - 0x92, 0x3, 0x92, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, - 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, - 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x95, 0x3, 0x95, 0x3, 0x95, - 0x3, 0x95, 0x3, 0x95, 0x3, 0x96, 0x3, 0x96, 0x3, 0x96, 0x3, 0x96, 0x3, - 0x96, 0x3, 0x96, 0x3, 0x96, 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, - 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, 0x3, 0x98, 0x3, 0x98, 0x3, 0x98, 0x3, - 0x98, 0x3, 0x98, 0x3, 0x98, 0x3, 0x99, 0x3, 0x99, 0x3, 0x99, 0x3, 0x99, - 0x3, 0x99, 0x3, 0x99, 0x3, 0x99, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, - 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, - 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9c, 0x3, - 0x9c, 0x3, 0x9c, 0x3, 0x9c, 0x3, 0x9c, 0x3, 0x9d, 0x3, 0x9d, 0x3, 0x9d, - 0x3, 0x9d, 0x3, 0x9d, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9e, 0x3, - 0x9e, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, - 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, - 0x9f, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa1, 0x3, 0xa1, 0x3, 0xa1, - 0x3, 0xa1, 0x3, 0xa2, 0x3, 0xa2, 0x3, 0xa2, 0x3, 0xa2, 0x3, 0xa2, 0x3, - 0xa2, 0x3, 0xa2, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, - 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa4, 0x3, 0xa4, 0x3, - 0xa4, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, - 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa6, 0x3, - 0xa6, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa7, 0x3, 0xa7, 0x3, 0xa7, 0x3, 0xa7, - 0x3, 0xa7, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, 0x3, - 0xa8, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, - 0x3, 0xa9, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xab, 0x3, - 0xab, 0x3, 0xab, 0x3, 0xab, 0x3, 0xab, 0x3, 0xab, 0x3, 0xac, 0x3, 0xac, - 0x3, 0xac, 0x3, 0xac, 0x3, 0xac, 0x3, 0xad, 0x3, 0xad, 0x3, 0xad, 0x3, - 0xad, 0x3, 0xad, 0x3, 0xad, 0x3, 0xad, 0x3, 0xae, 0x3, 0xae, 0x3, 0xae, - 0x3, 0xae, 0x3, 0xae, 0x3, 0xaf, 0x3, 0xaf, 0x3, 0xaf, 0x3, 0xaf, 0x3, - 0xaf, 0x3, 0xaf, 0x3, 0xaf, 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb0, - 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb1, 0x3, 0xb1, 0x3, 0xb1, 0x3, 0xb1, 0x3, - 0xb1, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb3, - 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb4, 0x3, - 0xb4, 0x3, 0xb4, 0x3, 0xb4, 0x3, 0xb4, 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb5, - 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb5, 0x3, - 0xb5, 0x5, 0xb5, 0x6b5, 0xa, 0xb5, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, - 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb7, 0x3, 0xb7, 0x3, 0xb7, 0x3, - 0xb7, 0x3, 0xb7, 0x3, 0xb8, 0x3, 0xb8, 0x5, 0xb8, 0x6c4, 0xa, 0xb8, - 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x7, 0xb8, 0x6c9, 0xa, 0xb8, 0xc, 0xb8, - 0xe, 0xb8, 0x6cc, 0xb, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, - 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x7, 0xb8, 0x6d6, 0xa, 0xb8, - 0xc, 0xb8, 0xe, 0xb8, 0x6d9, 0xb, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, - 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, - 0xb8, 0x7, 0xb8, 0x6e5, 0xa, 0xb8, 0xc, 0xb8, 0xe, 0xb8, 0x6e8, 0xb, - 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x5, 0xb8, 0x6ec, 0xa, 0xb8, 0x3, 0xb9, - 0x3, 0xb9, 0x3, 0xb9, 0x7, 0xb9, 0x6f1, 0xa, 0xb9, 0xc, 0xb9, 0xe, 0xb9, - 0x6f4, 0xb, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x6f8, 0xa, 0xb9, - 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x6fc, 0xa, 0xb9, 0x3, 0xb9, 0x6, 0xb9, - 0x6ff, 0xa, 0xb9, 0xd, 0xb9, 0xe, 0xb9, 0x700, 0x3, 0xb9, 0x3, 0xb9, - 0x3, 0xb9, 0x5, 0xb9, 0x706, 0xa, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, - 0x70a, 0xa, 0xb9, 0x3, 0xb9, 0x6, 0xb9, 0x70d, 0xa, 0xb9, 0xd, 0xb9, - 0xe, 0xb9, 0x70e, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x7, 0xb9, 0x714, - 0xa, 0xb9, 0xc, 0xb9, 0xe, 0xb9, 0x717, 0xb, 0xb9, 0x3, 0xb9, 0x3, 0xb9, - 0x3, 0xb9, 0x5, 0xb9, 0x71c, 0xa, 0xb9, 0x3, 0xb9, 0x6, 0xb9, 0x71f, - 0xa, 0xb9, 0xd, 0xb9, 0xe, 0xb9, 0x720, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, - 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x728, 0xa, 0xb9, 0x3, 0xb9, 0x6, 0xb9, - 0x72b, 0xa, 0xb9, 0xd, 0xb9, 0xe, 0xb9, 0x72c, 0x3, 0xb9, 0x3, 0xb9, - 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x733, 0xa, 0xb9, 0x3, 0xb9, 0x6, 0xb9, - 0x736, 0xa, 0xb9, 0xd, 0xb9, 0xe, 0xb9, 0x737, 0x5, 0xb9, 0x73a, 0xa, - 0xb9, 0x3, 0xba, 0x3, 0xba, 0x6, 0xba, 0x73e, 0xa, 0xba, 0xd, 0xba, - 0xe, 0xba, 0x73f, 0x3, 0xbb, 0x6, 0xbb, 0x743, 0xa, 0xbb, 0xd, 0xbb, - 0xe, 0xbb, 0x744, 0x3, 0xbc, 0x3, 0xbc, 0x3, 0xbc, 0x6, 0xbc, 0x74a, - 0xa, 0xbc, 0xd, 0xbc, 0xe, 0xbc, 0x74b, 0x3, 0xbd, 0x3, 0xbd, 0x3, 0xbd, - 0x3, 0xbd, 0x3, 0xbd, 0x3, 0xbd, 0x3, 0xbd, 0x3, 0xbd, 0x7, 0xbd, 0x756, - 0xa, 0xbd, 0xc, 0xbd, 0xe, 0xbd, 0x759, 0xb, 0xbd, 0x3, 0xbd, 0x3, 0xbd, - 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbf, 0x3, 0xbf, 0x3, 0xc0, 0x3, 0xc0, 0x3, - 0xc1, 0x3, 0xc1, 0x3, 0xc2, 0x3, 0xc2, 0x3, 0xc3, 0x3, 0xc3, 0x3, 0xc4, - 0x3, 0xc4, 0x3, 0xc5, 0x3, 0xc5, 0x3, 0xc6, 0x3, 0xc6, 0x3, 0xc7, 0x3, - 0xc7, 0x3, 0xc8, 0x3, 0xc8, 0x3, 0xc9, 0x3, 0xc9, 0x3, 0xca, 0x3, 0xca, - 0x3, 0xcb, 0x3, 0xcb, 0x3, 0xcc, 0x3, 0xcc, 0x3, 0xcd, 0x3, 0xcd, 0x3, - 0xce, 0x3, 0xce, 0x3, 0xcf, 0x3, 0xcf, 0x3, 0xd0, 0x3, 0xd0, 0x3, 0xd1, - 0x3, 0xd1, 0x3, 0xd2, 0x3, 0xd2, 0x3, 0xd3, 0x3, 0xd3, 0x3, 0xd4, 0x3, - 0xd4, 0x3, 0xd5, 0x3, 0xd5, 0x3, 0xd6, 0x3, 0xd6, 0x3, 0xd7, 0x3, 0xd7, - 0x3, 0xd8, 0x3, 0xd8, 0x3, 0xd9, 0x3, 0xd9, 0x3, 0xda, 0x3, 0xda, 0x3, - 0xdb, 0x3, 0xdb, 0x3, 0xdc, 0x3, 0xdc, 0x3, 0xdc, 0x3, 0xdd, 0x3, 0xdd, - 0x3, 0xde, 0x3, 0xde, 0x3, 0xdf, 0x3, 0xdf, 0x3, 0xe0, 0x3, 0xe0, 0x3, - 0xe1, 0x3, 0xe1, 0x3, 0xe2, 0x3, 0xe2, 0x3, 0xe2, 0x3, 0xe3, 0x3, 0xe3, - 0x3, 0xe4, 0x3, 0xe4, 0x3, 0xe5, 0x3, 0xe5, 0x3, 0xe5, 0x3, 0xe6, 0x3, - 0xe6, 0x3, 0xe7, 0x3, 0xe7, 0x3, 0xe7, 0x3, 0xe8, 0x3, 0xe8, 0x3, 0xe9, - 0x3, 0xe9, 0x3, 0xea, 0x3, 0xea, 0x3, 0xeb, 0x3, 0xeb, 0x3, 0xeb, 0x3, - 0xec, 0x3, 0xec, 0x3, 0xed, 0x3, 0xed, 0x3, 0xee, 0x3, 0xee, 0x3, 0xee, - 0x3, 0xee, 0x5, 0xee, 0x7c6, 0xa, 0xee, 0x3, 0xef, 0x3, 0xef, 0x3, 0xf0, - 0x3, 0xf0, 0x3, 0xf1, 0x3, 0xf1, 0x3, 0xf2, 0x3, 0xf2, 0x3, 0xf3, 0x3, - 0xf3, 0x3, 0xf4, 0x3, 0xf4, 0x3, 0xf5, 0x3, 0xf5, 0x3, 0xf6, 0x3, 0xf6, - 0x3, 0xf7, 0x3, 0xf7, 0x3, 0xf8, 0x3, 0xf8, 0x3, 0xf9, 0x3, 0xf9, 0x3, - 0xfa, 0x3, 0xfa, 0x3, 0xfa, 0x3, 0xfa, 0x7, 0xfa, 0x7e2, 0xa, 0xfa, - 0xc, 0xfa, 0xe, 0xfa, 0x7e5, 0xb, 0xfa, 0x3, 0xfa, 0x3, 0xfa, 0x3, 0xfa, - 0x3, 0xfa, 0x3, 0xfa, 0x3, 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x7, - 0xfb, 0x7f0, 0xa, 0xfb, 0xc, 0xfb, 0xe, 0xfb, 0x7f3, 0xb, 0xfb, 0x3, - 0xfb, 0x5, 0xfb, 0x7f6, 0xa, 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x3, 0xfc, - 0x3, 0xfc, 0x3, 0xfc, 0x3, 0xfc, 0x3, 0x7e3, 0x2, 0xfd, 0x3, 0x3, 0x5, - 0x4, 0x7, 0x5, 0x9, 0x6, 0xb, 0x7, 0xd, 0x8, 0xf, 0x9, 0x11, 0xa, 0x13, - 0xb, 0x15, 0xc, 0x17, 0xd, 0x19, 0xe, 0x1b, 0xf, 0x1d, 0x10, 0x1f, 0x11, - 0x21, 0x12, 0x23, 0x13, 0x25, 0x14, 0x27, 0x15, 0x29, 0x16, 0x2b, 0x17, - 0x2d, 0x18, 0x2f, 0x19, 0x31, 0x1a, 0x33, 0x1b, 0x35, 0x1c, 0x37, 0x1d, - 0x39, 0x1e, 0x3b, 0x1f, 0x3d, 0x20, 0x3f, 0x21, 0x41, 0x22, 0x43, 0x23, - 0x45, 0x24, 0x47, 0x25, 0x49, 0x26, 0x4b, 0x27, 0x4d, 0x28, 0x4f, 0x29, - 0x51, 0x2a, 0x53, 0x2b, 0x55, 0x2c, 0x57, 0x2d, 0x59, 0x2e, 0x5b, 0x2f, - 0x5d, 0x30, 0x5f, 0x31, 0x61, 0x32, 0x63, 0x33, 0x65, 0x34, 0x67, 0x35, - 0x69, 0x36, 0x6b, 0x37, 0x6d, 0x38, 0x6f, 0x39, 0x71, 0x3a, 0x73, 0x3b, - 0x75, 0x3c, 0x77, 0x3d, 0x79, 0x3e, 0x7b, 0x3f, 0x7d, 0x40, 0x7f, 0x41, - 0x81, 0x42, 0x83, 0x43, 0x85, 0x44, 0x87, 0x45, 0x89, 0x46, 0x8b, 0x47, - 0x8d, 0x48, 0x8f, 0x49, 0x91, 0x4a, 0x93, 0x4b, 0x95, 0x4c, 0x97, 0x4d, - 0x99, 0x4e, 0x9b, 0x4f, 0x9d, 0x50, 0x9f, 0x51, 0xa1, 0x52, 0xa3, 0x53, - 0xa5, 0x54, 0xa7, 0x55, 0xa9, 0x56, 0xab, 0x57, 0xad, 0x58, 0xaf, 0x59, - 0xb1, 0x5a, 0xb3, 0x5b, 0xb5, 0x5c, 0xb7, 0x5d, 0xb9, 0x5e, 0xbb, 0x5f, - 0xbd, 0x60, 0xbf, 0x61, 0xc1, 0x62, 0xc3, 0x63, 0xc5, 0x64, 0xc7, 0x65, - 0xc9, 0x66, 0xcb, 0x67, 0xcd, 0x68, 0xcf, 0x69, 0xd1, 0x6a, 0xd3, 0x6b, - 0xd5, 0x6c, 0xd7, 0x6d, 0xd9, 0x6e, 0xdb, 0x6f, 0xdd, 0x70, 0xdf, 0x71, - 0xe1, 0x72, 0xe3, 0x73, 0xe5, 0x74, 0xe7, 0x75, 0xe9, 0x76, 0xeb, 0x77, - 0xed, 0x78, 0xef, 0x79, 0xf1, 0x7a, 0xf3, 0x7b, 0xf5, 0x7c, 0xf7, 0x7d, - 0xf9, 0x7e, 0xfb, 0x7f, 0xfd, 0x80, 0xff, 0x81, 0x101, 0x82, 0x103, - 0x83, 0x105, 0x84, 0x107, 0x85, 0x109, 0x86, 0x10b, 0x87, 0x10d, 0x88, - 0x10f, 0x89, 0x111, 0x8a, 0x113, 0x8b, 0x115, 0x8c, 0x117, 0x8d, 0x119, - 0x8e, 0x11b, 0x8f, 0x11d, 0x90, 0x11f, 0x91, 0x121, 0x92, 0x123, 0x93, - 0x125, 0x94, 0x127, 0x95, 0x129, 0x96, 0x12b, 0x97, 0x12d, 0x98, 0x12f, - 0x99, 0x131, 0x9a, 0x133, 0x9b, 0x135, 0x9c, 0x137, 0x9d, 0x139, 0x9e, - 0x13b, 0x9f, 0x13d, 0xa0, 0x13f, 0xa1, 0x141, 0xa2, 0x143, 0xa3, 0x145, - 0xa4, 0x147, 0xa5, 0x149, 0xa6, 0x14b, 0xa7, 0x14d, 0xa8, 0x14f, 0xa9, - 0x151, 0xaa, 0x153, 0xab, 0x155, 0xac, 0x157, 0xad, 0x159, 0xae, 0x15b, - 0xaf, 0x15d, 0xb0, 0x15f, 0xb1, 0x161, 0xb2, 0x163, 0xb3, 0x165, 0xb4, - 0x167, 0xb5, 0x169, 0xb6, 0x16b, 0xb7, 0x16d, 0xb8, 0x16f, 0xb9, 0x171, - 0xba, 0x173, 0xbb, 0x175, 0xbc, 0x177, 0xbd, 0x179, 0xbe, 0x17b, 0x2, - 0x17d, 0x2, 0x17f, 0x2, 0x181, 0x2, 0x183, 0x2, 0x185, 0x2, 0x187, 0x2, - 0x189, 0x2, 0x18b, 0x2, 0x18d, 0x2, 0x18f, 0x2, 0x191, 0x2, 0x193, 0x2, - 0x195, 0x2, 0x197, 0x2, 0x199, 0x2, 0x19b, 0x2, 0x19d, 0x2, 0x19f, 0x2, - 0x1a1, 0x2, 0x1a3, 0x2, 0x1a5, 0x2, 0x1a7, 0x2, 0x1a9, 0x2, 0x1ab, 0x2, - 0x1ad, 0x2, 0x1af, 0x2, 0x1b1, 0x2, 0x1b3, 0x2, 0x1b5, 0x2, 0x1b7, 0xbf, - 0x1b9, 0xc0, 0x1bb, 0xc1, 0x1bd, 0xc2, 0x1bf, 0xc3, 0x1c1, 0xc4, 0x1c3, - 0xc5, 0x1c5, 0xc6, 0x1c7, 0xc7, 0x1c9, 0xc8, 0x1cb, 0xc9, 0x1cd, 0xca, - 0x1cf, 0xcb, 0x1d1, 0xcc, 0x1d3, 0xcd, 0x1d5, 0xce, 0x1d7, 0xcf, 0x1d9, - 0xd0, 0x1db, 0xd1, 0x1dd, 0xd2, 0x1df, 0xd3, 0x1e1, 0xd4, 0x1e3, 0xd5, - 0x1e5, 0xd6, 0x1e7, 0xd7, 0x1e9, 0xd8, 0x1eb, 0xd9, 0x1ed, 0xda, 0x1ef, - 0xdb, 0x1f1, 0xdc, 0x1f3, 0xdd, 0x1f5, 0xde, 0x1f7, 0xdf, 0x3, 0x2, + 0x3, 0x7e, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, + 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x7f, 0x3, 0x80, + 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, 0x80, 0x3, + 0x80, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, 0x3, 0x81, + 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, 0x82, 0x3, + 0x82, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, 0x3, 0x83, + 0x3, 0x83, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, 0x84, 0x3, + 0x84, 0x3, 0x84, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, + 0x3, 0x85, 0x3, 0x85, 0x3, 0x85, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, + 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x86, 0x3, 0x87, 0x3, 0x87, + 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, + 0x87, 0x3, 0x87, 0x3, 0x87, 0x3, 0x88, 0x3, 0x88, 0x3, 0x88, 0x3, 0x88, + 0x3, 0x88, 0x3, 0x88, 0x3, 0x89, 0x3, 0x89, 0x3, 0x89, 0x3, 0x89, 0x3, + 0x89, 0x3, 0x89, 0x3, 0x89, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8a, + 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8a, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, + 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8b, 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8c, + 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8c, 0x3, 0x8d, 0x3, 0x8d, 0x3, + 0x8d, 0x3, 0x8d, 0x3, 0x8d, 0x3, 0x8e, 0x3, 0x8e, 0x3, 0x8e, 0x3, 0x8e, + 0x3, 0x8e, 0x3, 0x8e, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, 0x8f, 0x3, + 0x90, 0x3, 0x90, 0x3, 0x90, 0x3, 0x90, 0x3, 0x90, 0x3, 0x90, 0x3, 0x90, + 0x3, 0x90, 0x3, 0x90, 0x3, 0x91, 0x3, 0x91, 0x3, 0x91, 0x3, 0x91, 0x3, + 0x91, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, 0x3, 0x92, + 0x3, 0x92, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, 0x3, 0x93, 0x3, + 0x93, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x94, 0x3, 0x95, + 0x3, 0x95, 0x3, 0x95, 0x3, 0x95, 0x3, 0x95, 0x3, 0x95, 0x3, 0x95, 0x3, + 0x95, 0x3, 0x95, 0x3, 0x95, 0x3, 0x96, 0x3, 0x96, 0x3, 0x96, 0x3, 0x96, + 0x3, 0x96, 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, 0x3, 0x97, 0x3, + 0x97, 0x3, 0x97, 0x3, 0x98, 0x3, 0x98, 0x3, 0x98, 0x3, 0x98, 0x3, 0x98, + 0x3, 0x98, 0x3, 0x98, 0x3, 0x99, 0x3, 0x99, 0x3, 0x99, 0x3, 0x99, 0x3, + 0x99, 0x3, 0x99, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9a, + 0x3, 0x9a, 0x3, 0x9a, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, + 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9b, 0x3, 0x9c, + 0x3, 0x9c, 0x3, 0x9c, 0x3, 0x9c, 0x3, 0x9c, 0x3, 0x9d, 0x3, 0x9d, 0x3, + 0x9d, 0x3, 0x9d, 0x3, 0x9d, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9e, 0x3, 0x9e, + 0x3, 0x9e, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, + 0x9f, 0x3, 0x9f, 0x3, 0x9f, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, + 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, 0xa0, 0x3, + 0xa1, 0x3, 0xa1, 0x3, 0xa1, 0x3, 0xa2, 0x3, 0xa2, 0x3, 0xa2, 0x3, 0xa2, + 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, 0xa3, 0x3, + 0xa3, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa4, + 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa4, 0x3, 0xa5, 0x3, 0xa5, 0x3, 0xa5, 0x3, + 0xa5, 0x3, 0xa5, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa6, + 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa6, 0x3, 0xa7, 0x3, 0xa7, 0x3, + 0xa7, 0x3, 0xa7, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, 0x3, 0xa8, + 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, 0xa9, 0x3, + 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, 0x3, 0xaa, + 0x3, 0xab, 0x3, 0xab, 0x3, 0xab, 0x3, 0xab, 0x3, 0xac, 0x3, 0xac, 0x3, + 0xac, 0x3, 0xac, 0x3, 0xac, 0x3, 0xac, 0x3, 0xad, 0x3, 0xad, 0x3, 0xad, + 0x3, 0xad, 0x3, 0xad, 0x3, 0xae, 0x3, 0xae, 0x3, 0xae, 0x3, 0xae, 0x3, + 0xae, 0x3, 0xae, 0x3, 0xae, 0x3, 0xaf, 0x3, 0xaf, 0x3, 0xaf, 0x3, 0xaf, + 0x3, 0xaf, 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb0, 0x3, 0xb0, 0x3, + 0xb0, 0x3, 0xb0, 0x3, 0xb1, 0x3, 0xb1, 0x3, 0xb1, 0x3, 0xb1, 0x3, 0xb1, + 0x3, 0xb1, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, 0xb2, 0x3, + 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb3, 0x3, 0xb4, 0x3, 0xb4, + 0x3, 0xb4, 0x3, 0xb4, 0x3, 0xb4, 0x3, 0xb4, 0x3, 0xb5, 0x3, 0xb5, 0x3, + 0xb5, 0x3, 0xb5, 0x3, 0xb5, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, + 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x3, 0xb6, 0x5, + 0xb6, 0x6bb, 0xa, 0xb6, 0x3, 0xb7, 0x3, 0xb7, 0x3, 0xb7, 0x3, 0xb7, + 0x3, 0xb7, 0x3, 0xb7, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, 0xb8, 0x3, + 0xb8, 0x3, 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x6ca, 0xa, 0xb9, 0x3, 0xb9, + 0x3, 0xb9, 0x3, 0xb9, 0x7, 0xb9, 0x6cf, 0xa, 0xb9, 0xc, 0xb9, 0xe, 0xb9, + 0x6d2, 0xb, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, + 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x7, 0xb9, 0x6dc, 0xa, 0xb9, 0xc, 0xb9, + 0xe, 0xb9, 0x6df, 0xb, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, + 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x3, 0xb9, 0x7, + 0xb9, 0x6eb, 0xa, 0xb9, 0xc, 0xb9, 0xe, 0xb9, 0x6ee, 0xb, 0xb9, 0x3, + 0xb9, 0x3, 0xb9, 0x5, 0xb9, 0x6f2, 0xa, 0xb9, 0x3, 0xba, 0x3, 0xba, + 0x3, 0xba, 0x7, 0xba, 0x6f7, 0xa, 0xba, 0xc, 0xba, 0xe, 0xba, 0x6fa, + 0xb, 0xba, 0x3, 0xba, 0x3, 0xba, 0x5, 0xba, 0x6fe, 0xa, 0xba, 0x3, 0xba, + 0x3, 0xba, 0x5, 0xba, 0x702, 0xa, 0xba, 0x3, 0xba, 0x6, 0xba, 0x705, + 0xa, 0xba, 0xd, 0xba, 0xe, 0xba, 0x706, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, + 0x5, 0xba, 0x70c, 0xa, 0xba, 0x3, 0xba, 0x3, 0xba, 0x5, 0xba, 0x710, + 0xa, 0xba, 0x3, 0xba, 0x6, 0xba, 0x713, 0xa, 0xba, 0xd, 0xba, 0xe, 0xba, + 0x714, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, 0x7, 0xba, 0x71a, 0xa, 0xba, + 0xc, 0xba, 0xe, 0xba, 0x71d, 0xb, 0xba, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, + 0x5, 0xba, 0x722, 0xa, 0xba, 0x3, 0xba, 0x6, 0xba, 0x725, 0xa, 0xba, + 0xd, 0xba, 0xe, 0xba, 0x726, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, + 0x3, 0xba, 0x5, 0xba, 0x72e, 0xa, 0xba, 0x3, 0xba, 0x6, 0xba, 0x731, + 0xa, 0xba, 0xd, 0xba, 0xe, 0xba, 0x732, 0x3, 0xba, 0x3, 0xba, 0x3, 0xba, + 0x3, 0xba, 0x5, 0xba, 0x739, 0xa, 0xba, 0x3, 0xba, 0x6, 0xba, 0x73c, + 0xa, 0xba, 0xd, 0xba, 0xe, 0xba, 0x73d, 0x5, 0xba, 0x740, 0xa, 0xba, + 0x3, 0xbb, 0x3, 0xbb, 0x6, 0xbb, 0x744, 0xa, 0xbb, 0xd, 0xbb, 0xe, 0xbb, + 0x745, 0x3, 0xbc, 0x6, 0xbc, 0x749, 0xa, 0xbc, 0xd, 0xbc, 0xe, 0xbc, + 0x74a, 0x3, 0xbd, 0x3, 0xbd, 0x3, 0xbd, 0x6, 0xbd, 0x750, 0xa, 0xbd, + 0xd, 0xbd, 0xe, 0xbd, 0x751, 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbe, + 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbe, 0x7, 0xbe, 0x75c, 0xa, 0xbe, + 0xc, 0xbe, 0xe, 0xbe, 0x75f, 0xb, 0xbe, 0x3, 0xbe, 0x3, 0xbe, 0x3, 0xbf, + 0x3, 0xbf, 0x3, 0xc0, 0x3, 0xc0, 0x3, 0xc1, 0x3, 0xc1, 0x3, 0xc2, 0x3, + 0xc2, 0x3, 0xc3, 0x3, 0xc3, 0x3, 0xc4, 0x3, 0xc4, 0x3, 0xc5, 0x3, 0xc5, + 0x3, 0xc6, 0x3, 0xc6, 0x3, 0xc7, 0x3, 0xc7, 0x3, 0xc8, 0x3, 0xc8, 0x3, + 0xc9, 0x3, 0xc9, 0x3, 0xca, 0x3, 0xca, 0x3, 0xcb, 0x3, 0xcb, 0x3, 0xcc, + 0x3, 0xcc, 0x3, 0xcd, 0x3, 0xcd, 0x3, 0xce, 0x3, 0xce, 0x3, 0xcf, 0x3, + 0xcf, 0x3, 0xd0, 0x3, 0xd0, 0x3, 0xd1, 0x3, 0xd1, 0x3, 0xd2, 0x3, 0xd2, + 0x3, 0xd3, 0x3, 0xd3, 0x3, 0xd4, 0x3, 0xd4, 0x3, 0xd5, 0x3, 0xd5, 0x3, + 0xd6, 0x3, 0xd6, 0x3, 0xd7, 0x3, 0xd7, 0x3, 0xd8, 0x3, 0xd8, 0x3, 0xd9, + 0x3, 0xd9, 0x3, 0xda, 0x3, 0xda, 0x3, 0xdb, 0x3, 0xdb, 0x3, 0xdc, 0x3, + 0xdc, 0x3, 0xdd, 0x3, 0xdd, 0x3, 0xdd, 0x3, 0xde, 0x3, 0xde, 0x3, 0xdf, + 0x3, 0xdf, 0x3, 0xe0, 0x3, 0xe0, 0x3, 0xe1, 0x3, 0xe1, 0x3, 0xe2, 0x3, + 0xe2, 0x3, 0xe3, 0x3, 0xe3, 0x3, 0xe3, 0x3, 0xe4, 0x3, 0xe4, 0x3, 0xe5, + 0x3, 0xe5, 0x3, 0xe6, 0x3, 0xe6, 0x3, 0xe6, 0x3, 0xe7, 0x3, 0xe7, 0x3, + 0xe8, 0x3, 0xe8, 0x3, 0xe8, 0x3, 0xe9, 0x3, 0xe9, 0x3, 0xea, 0x3, 0xea, + 0x3, 0xeb, 0x3, 0xeb, 0x3, 0xec, 0x3, 0xec, 0x3, 0xec, 0x3, 0xed, 0x3, + 0xed, 0x3, 0xee, 0x3, 0xee, 0x3, 0xef, 0x3, 0xef, 0x3, 0xef, 0x3, 0xef, + 0x5, 0xef, 0x7cc, 0xa, 0xef, 0x3, 0xf0, 0x3, 0xf0, 0x3, 0xf1, 0x3, 0xf1, + 0x3, 0xf2, 0x3, 0xf2, 0x3, 0xf3, 0x3, 0xf3, 0x3, 0xf4, 0x3, 0xf4, 0x3, + 0xf5, 0x3, 0xf5, 0x3, 0xf6, 0x3, 0xf6, 0x3, 0xf7, 0x3, 0xf7, 0x3, 0xf8, + 0x3, 0xf8, 0x3, 0xf9, 0x3, 0xf9, 0x3, 0xfa, 0x3, 0xfa, 0x3, 0xfb, 0x3, + 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x7, 0xfb, 0x7e8, 0xa, 0xfb, 0xc, 0xfb, + 0xe, 0xfb, 0x7eb, 0xb, 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x3, 0xfb, 0x3, 0xfb, + 0x3, 0xfb, 0x3, 0xfc, 0x3, 0xfc, 0x3, 0xfc, 0x3, 0xfc, 0x7, 0xfc, 0x7f6, + 0xa, 0xfc, 0xc, 0xfc, 0xe, 0xfc, 0x7f9, 0xb, 0xfc, 0x3, 0xfc, 0x5, 0xfc, + 0x7fc, 0xa, 0xfc, 0x3, 0xfc, 0x3, 0xfc, 0x3, 0xfd, 0x3, 0xfd, 0x3, 0xfd, + 0x3, 0xfd, 0x3, 0x7e9, 0x2, 0xfe, 0x3, 0x3, 0x5, 0x4, 0x7, 0x5, 0x9, + 0x6, 0xb, 0x7, 0xd, 0x8, 0xf, 0x9, 0x11, 0xa, 0x13, 0xb, 0x15, 0xc, + 0x17, 0xd, 0x19, 0xe, 0x1b, 0xf, 0x1d, 0x10, 0x1f, 0x11, 0x21, 0x12, + 0x23, 0x13, 0x25, 0x14, 0x27, 0x15, 0x29, 0x16, 0x2b, 0x17, 0x2d, 0x18, + 0x2f, 0x19, 0x31, 0x1a, 0x33, 0x1b, 0x35, 0x1c, 0x37, 0x1d, 0x39, 0x1e, + 0x3b, 0x1f, 0x3d, 0x20, 0x3f, 0x21, 0x41, 0x22, 0x43, 0x23, 0x45, 0x24, + 0x47, 0x25, 0x49, 0x26, 0x4b, 0x27, 0x4d, 0x28, 0x4f, 0x29, 0x51, 0x2a, + 0x53, 0x2b, 0x55, 0x2c, 0x57, 0x2d, 0x59, 0x2e, 0x5b, 0x2f, 0x5d, 0x30, + 0x5f, 0x31, 0x61, 0x32, 0x63, 0x33, 0x65, 0x34, 0x67, 0x35, 0x69, 0x36, + 0x6b, 0x37, 0x6d, 0x38, 0x6f, 0x39, 0x71, 0x3a, 0x73, 0x3b, 0x75, 0x3c, + 0x77, 0x3d, 0x79, 0x3e, 0x7b, 0x3f, 0x7d, 0x40, 0x7f, 0x41, 0x81, 0x42, + 0x83, 0x43, 0x85, 0x44, 0x87, 0x45, 0x89, 0x46, 0x8b, 0x47, 0x8d, 0x48, + 0x8f, 0x49, 0x91, 0x4a, 0x93, 0x4b, 0x95, 0x4c, 0x97, 0x4d, 0x99, 0x4e, + 0x9b, 0x4f, 0x9d, 0x50, 0x9f, 0x51, 0xa1, 0x52, 0xa3, 0x53, 0xa5, 0x54, + 0xa7, 0x55, 0xa9, 0x56, 0xab, 0x57, 0xad, 0x58, 0xaf, 0x59, 0xb1, 0x5a, + 0xb3, 0x5b, 0xb5, 0x5c, 0xb7, 0x5d, 0xb9, 0x5e, 0xbb, 0x5f, 0xbd, 0x60, + 0xbf, 0x61, 0xc1, 0x62, 0xc3, 0x63, 0xc5, 0x64, 0xc7, 0x65, 0xc9, 0x66, + 0xcb, 0x67, 0xcd, 0x68, 0xcf, 0x69, 0xd1, 0x6a, 0xd3, 0x6b, 0xd5, 0x6c, + 0xd7, 0x6d, 0xd9, 0x6e, 0xdb, 0x6f, 0xdd, 0x70, 0xdf, 0x71, 0xe1, 0x72, + 0xe3, 0x73, 0xe5, 0x74, 0xe7, 0x75, 0xe9, 0x76, 0xeb, 0x77, 0xed, 0x78, + 0xef, 0x79, 0xf1, 0x7a, 0xf3, 0x7b, 0xf5, 0x7c, 0xf7, 0x7d, 0xf9, 0x7e, + 0xfb, 0x7f, 0xfd, 0x80, 0xff, 0x81, 0x101, 0x82, 0x103, 0x83, 0x105, + 0x84, 0x107, 0x85, 0x109, 0x86, 0x10b, 0x87, 0x10d, 0x88, 0x10f, 0x89, + 0x111, 0x8a, 0x113, 0x8b, 0x115, 0x8c, 0x117, 0x8d, 0x119, 0x8e, 0x11b, + 0x8f, 0x11d, 0x90, 0x11f, 0x91, 0x121, 0x92, 0x123, 0x93, 0x125, 0x94, + 0x127, 0x95, 0x129, 0x96, 0x12b, 0x97, 0x12d, 0x98, 0x12f, 0x99, 0x131, + 0x9a, 0x133, 0x9b, 0x135, 0x9c, 0x137, 0x9d, 0x139, 0x9e, 0x13b, 0x9f, + 0x13d, 0xa0, 0x13f, 0xa1, 0x141, 0xa2, 0x143, 0xa3, 0x145, 0xa4, 0x147, + 0xa5, 0x149, 0xa6, 0x14b, 0xa7, 0x14d, 0xa8, 0x14f, 0xa9, 0x151, 0xaa, + 0x153, 0xab, 0x155, 0xac, 0x157, 0xad, 0x159, 0xae, 0x15b, 0xaf, 0x15d, + 0xb0, 0x15f, 0xb1, 0x161, 0xb2, 0x163, 0xb3, 0x165, 0xb4, 0x167, 0xb5, + 0x169, 0xb6, 0x16b, 0xb7, 0x16d, 0xb8, 0x16f, 0xb9, 0x171, 0xba, 0x173, + 0xbb, 0x175, 0xbc, 0x177, 0xbd, 0x179, 0xbe, 0x17b, 0xbf, 0x17d, 0x2, + 0x17f, 0x2, 0x181, 0x2, 0x183, 0x2, 0x185, 0x2, 0x187, 0x2, 0x189, 0x2, + 0x18b, 0x2, 0x18d, 0x2, 0x18f, 0x2, 0x191, 0x2, 0x193, 0x2, 0x195, 0x2, + 0x197, 0x2, 0x199, 0x2, 0x19b, 0x2, 0x19d, 0x2, 0x19f, 0x2, 0x1a1, 0x2, + 0x1a3, 0x2, 0x1a5, 0x2, 0x1a7, 0x2, 0x1a9, 0x2, 0x1ab, 0x2, 0x1ad, 0x2, + 0x1af, 0x2, 0x1b1, 0x2, 0x1b3, 0x2, 0x1b5, 0x2, 0x1b7, 0x2, 0x1b9, 0xc0, + 0x1bb, 0xc1, 0x1bd, 0xc2, 0x1bf, 0xc3, 0x1c1, 0xc4, 0x1c3, 0xc5, 0x1c5, + 0xc6, 0x1c7, 0xc7, 0x1c9, 0xc8, 0x1cb, 0xc9, 0x1cd, 0xca, 0x1cf, 0xcb, + 0x1d1, 0xcc, 0x1d3, 0xcd, 0x1d5, 0xce, 0x1d7, 0xcf, 0x1d9, 0xd0, 0x1db, + 0xd1, 0x1dd, 0xd2, 0x1df, 0xd3, 0x1e1, 0xd4, 0x1e3, 0xd5, 0x1e5, 0xd6, + 0x1e7, 0xd7, 0x1e9, 0xd8, 0x1eb, 0xd9, 0x1ed, 0xda, 0x1ef, 0xdb, 0x1f1, + 0xdc, 0x1f3, 0xdd, 0x1f5, 0xde, 0x1f7, 0xdf, 0x1f9, 0xe0, 0x3, 0x2, 0x26, 0x4, 0x2, 0x5e, 0x5e, 0x62, 0x62, 0x4, 0x2, 0x24, 0x24, 0x5e, 0x5e, 0x4, 0x2, 0x29, 0x29, 0x5e, 0x5e, 0x4, 0x2, 0x43, 0x43, 0x63, 0x63, 0x4, 0x2, 0x44, 0x44, 0x64, 0x64, 0x4, 0x2, 0x45, 0x45, 0x65, @@ -567,7 +568,7 @@ ClickHouseLexer::Initializer::Initializer() { 0x7b, 0x4, 0x2, 0x5c, 0x5c, 0x7c, 0x7c, 0x4, 0x2, 0x43, 0x5c, 0x63, 0x7c, 0x3, 0x2, 0x32, 0x39, 0x3, 0x2, 0x32, 0x3b, 0x5, 0x2, 0x32, 0x3b, 0x43, 0x48, 0x63, 0x68, 0x4, 0x2, 0xc, 0xc, 0xf, 0xf, 0x4, 0x3, 0xc, - 0xc, 0xf, 0xf, 0x4, 0x2, 0xb, 0xf, 0x22, 0x22, 0x2, 0x80d, 0x2, 0x3, + 0xc, 0xf, 0xf, 0x4, 0x2, 0xb, 0xf, 0x22, 0x22, 0x2, 0x813, 0x2, 0x3, 0x3, 0x2, 0x2, 0x2, 0x2, 0x5, 0x3, 0x2, 0x2, 0x2, 0x2, 0x7, 0x3, 0x2, 0x2, 0x2, 0x2, 0x9, 0x3, 0x2, 0x2, 0x2, 0x2, 0xb, 0x3, 0x2, 0x2, 0x2, 0x2, 0xd, 0x3, 0x2, 0x2, 0x2, 0x2, 0xf, 0x3, 0x2, 0x2, 0x2, 0x2, 0x11, @@ -650,7 +651,7 @@ ClickHouseLexer::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x2, 0x16d, 0x3, 0x2, 0x2, 0x2, 0x2, 0x16f, 0x3, 0x2, 0x2, 0x2, 0x2, 0x171, 0x3, 0x2, 0x2, 0x2, 0x2, 0x173, 0x3, 0x2, 0x2, 0x2, 0x2, 0x175, 0x3, 0x2, 0x2, 0x2, 0x2, 0x177, 0x3, 0x2, 0x2, 0x2, - 0x2, 0x179, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1b7, 0x3, 0x2, 0x2, 0x2, 0x2, + 0x2, 0x179, 0x3, 0x2, 0x2, 0x2, 0x2, 0x17b, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1b9, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1bb, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1bd, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1bf, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1c1, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1c3, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1c5, 0x3, 0x2, @@ -665,925 +666,928 @@ ClickHouseLexer::Initializer::Initializer() { 0x2, 0x1e9, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1eb, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1ed, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1ef, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1f1, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1f3, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1f5, 0x3, - 0x2, 0x2, 0x2, 0x2, 0x1f7, 0x3, 0x2, 0x2, 0x2, 0x3, 0x1f9, 0x3, 0x2, - 0x2, 0x2, 0x5, 0x1fd, 0x3, 0x2, 0x2, 0x2, 0x7, 0x203, 0x3, 0x2, 0x2, - 0x2, 0x9, 0x209, 0x3, 0x2, 0x2, 0x2, 0xb, 0x20d, 0x3, 0x2, 0x2, 0x2, - 0xd, 0x213, 0x3, 0x2, 0x2, 0x2, 0xf, 0x217, 0x3, 0x2, 0x2, 0x2, 0x11, - 0x21c, 0x3, 0x2, 0x2, 0x2, 0x13, 0x220, 0x3, 0x2, 0x2, 0x2, 0x15, 0x226, - 0x3, 0x2, 0x2, 0x2, 0x17, 0x237, 0x3, 0x2, 0x2, 0x2, 0x19, 0x239, 0x3, - 0x2, 0x2, 0x2, 0x1b, 0x23e, 0x3, 0x2, 0x2, 0x2, 0x1d, 0x244, 0x3, 0x2, - 0x2, 0x2, 0x1f, 0x24b, 0x3, 0x2, 0x2, 0x2, 0x21, 0x253, 0x3, 0x2, 0x2, - 0x2, 0x23, 0x258, 0x3, 0x2, 0x2, 0x2, 0x25, 0x25b, 0x3, 0x2, 0x2, 0x2, - 0x27, 0x260, 0x3, 0x2, 0x2, 0x2, 0x29, 0x265, 0x3, 0x2, 0x2, 0x2, 0x2b, - 0x26b, 0x3, 0x2, 0x2, 0x2, 0x2d, 0x271, 0x3, 0x2, 0x2, 0x2, 0x2f, 0x279, - 0x3, 0x2, 0x2, 0x2, 0x31, 0x27f, 0x3, 0x2, 0x2, 0x2, 0x33, 0x287, 0x3, - 0x2, 0x2, 0x2, 0x35, 0x28e, 0x3, 0x2, 0x2, 0x2, 0x37, 0x296, 0x3, 0x2, - 0x2, 0x2, 0x39, 0x2a1, 0x3, 0x2, 0x2, 0x2, 0x3b, 0x2a8, 0x3, 0x2, 0x2, - 0x2, 0x3d, 0x2ae, 0x3, 0x2, 0x2, 0x2, 0x3f, 0x2b3, 0x3, 0x2, 0x2, 0x2, - 0x41, 0x2bc, 0x3, 0x2, 0x2, 0x2, 0x43, 0x2c6, 0x3, 0x2, 0x2, 0x2, 0x45, - 0x2cb, 0x3, 0x2, 0x2, 0x2, 0x47, 0x2cf, 0x3, 0x2, 0x2, 0x2, 0x49, 0x2db, - 0x3, 0x2, 0x2, 0x2, 0x4b, 0x2e3, 0x3, 0x2, 0x2, 0x2, 0x4d, 0x2e9, 0x3, - 0x2, 0x2, 0x2, 0x4f, 0x2f0, 0x3, 0x2, 0x2, 0x2, 0x51, 0x2f5, 0x3, 0x2, - 0x2, 0x2, 0x53, 0x300, 0x3, 0x2, 0x2, 0x2, 0x55, 0x309, 0x3, 0x2, 0x2, - 0x2, 0x57, 0x310, 0x3, 0x2, 0x2, 0x2, 0x59, 0x31d, 0x3, 0x2, 0x2, 0x2, - 0x5b, 0x328, 0x3, 0x2, 0x2, 0x2, 0x5d, 0x32d, 0x3, 0x2, 0x2, 0x2, 0x5f, - 0x336, 0x3, 0x2, 0x2, 0x2, 0x61, 0x342, 0x3, 0x2, 0x2, 0x2, 0x63, 0x347, - 0x3, 0x2, 0x2, 0x2, 0x65, 0x34c, 0x3, 0x2, 0x2, 0x2, 0x67, 0x350, 0x3, - 0x2, 0x2, 0x2, 0x69, 0x357, 0x3, 0x2, 0x2, 0x2, 0x6b, 0x35e, 0x3, 0x2, - 0x2, 0x2, 0x6d, 0x365, 0x3, 0x2, 0x2, 0x2, 0x6f, 0x36d, 0x3, 0x2, 0x2, - 0x2, 0x71, 0x378, 0x3, 0x2, 0x2, 0x2, 0x73, 0x380, 0x3, 0x2, 0x2, 0x2, - 0x75, 0x388, 0x3, 0x2, 0x2, 0x2, 0x77, 0x38e, 0x3, 0x2, 0x2, 0x2, 0x79, - 0x394, 0x3, 0x2, 0x2, 0x2, 0x7b, 0x39a, 0x3, 0x2, 0x2, 0x2, 0x7d, 0x39e, - 0x3, 0x2, 0x2, 0x2, 0x7f, 0x3a5, 0x3, 0x2, 0x2, 0x2, 0x81, 0x3ac, 0x3, - 0x2, 0x2, 0x2, 0x83, 0x3b1, 0x3, 0x2, 0x2, 0x2, 0x85, 0x3b6, 0x3, 0x2, - 0x2, 0x2, 0x87, 0x3bf, 0x3, 0x2, 0x2, 0x2, 0x89, 0x3c6, 0x3, 0x2, 0x2, - 0x2, 0x8b, 0x3d2, 0x3, 0x2, 0x2, 0x2, 0x8d, 0x3d8, 0x3, 0x2, 0x2, 0x2, - 0x8f, 0x3df, 0x3, 0x2, 0x2, 0x2, 0x91, 0x3ec, 0x3, 0x2, 0x2, 0x2, 0x93, - 0x3f1, 0x3, 0x2, 0x2, 0x2, 0x95, 0x3f4, 0x3, 0x2, 0x2, 0x2, 0x97, 0x3f7, - 0x3, 0x2, 0x2, 0x2, 0x99, 0x3fd, 0x3, 0x2, 0x2, 0x2, 0x9b, 0x400, 0x3, - 0x2, 0x2, 0x2, 0x9d, 0x413, 0x3, 0x2, 0x2, 0x2, 0x9f, 0x415, 0x3, 0x2, - 0x2, 0x2, 0xa1, 0x41f, 0x3, 0x2, 0x2, 0x2, 0xa3, 0x425, 0x3, 0x2, 0x2, - 0x2, 0xa5, 0x42c, 0x3, 0x2, 0x2, 0x2, 0xa7, 0x435, 0x3, 0x2, 0x2, 0x2, - 0xa9, 0x43a, 0x3, 0x2, 0x2, 0x2, 0xab, 0x43d, 0x3, 0x2, 0x2, 0x2, 0xad, - 0x44a, 0x3, 0x2, 0x2, 0x2, 0xaf, 0x44f, 0x3, 0x2, 0x2, 0x2, 0xb1, 0x453, - 0x3, 0x2, 0x2, 0x2, 0xb3, 0x458, 0x3, 0x2, 0x2, 0x2, 0xb5, 0x45d, 0x3, - 0x2, 0x2, 0x2, 0xb7, 0x464, 0x3, 0x2, 0x2, 0x2, 0xb9, 0x46c, 0x3, 0x2, - 0x2, 0x2, 0xbb, 0x471, 0x3, 0x2, 0x2, 0x2, 0xbd, 0x47a, 0x3, 0x2, 0x2, - 0x2, 0xbf, 0x47f, 0x3, 0x2, 0x2, 0x2, 0xc1, 0x485, 0x3, 0x2, 0x2, 0x2, - 0xc3, 0x48a, 0x3, 0x2, 0x2, 0x2, 0xc5, 0x490, 0x3, 0x2, 0x2, 0x2, 0xc7, - 0x495, 0x3, 0x2, 0x2, 0x2, 0xc9, 0x4a2, 0x3, 0x2, 0x2, 0x2, 0xcb, 0x4ae, - 0x3, 0x2, 0x2, 0x2, 0xcd, 0x4b2, 0x3, 0x2, 0x2, 0x2, 0xcf, 0x4b9, 0x3, - 0x2, 0x2, 0x2, 0xd1, 0x4bd, 0x3, 0x2, 0x2, 0x2, 0xd3, 0x4c4, 0x3, 0x2, - 0x2, 0x2, 0xd5, 0x4cb, 0x3, 0x2, 0x2, 0x2, 0xd7, 0x4d1, 0x3, 0x2, 0x2, - 0x2, 0xd9, 0x4d6, 0x3, 0x2, 0x2, 0x2, 0xdb, 0x4df, 0x3, 0x2, 0x2, 0x2, - 0xdd, 0x4e3, 0x3, 0x2, 0x2, 0x2, 0xdf, 0x4e6, 0x3, 0x2, 0x2, 0x2, 0xe1, - 0x4ea, 0x3, 0x2, 0x2, 0x2, 0xe3, 0x4ef, 0x3, 0x2, 0x2, 0x2, 0xe5, 0x4f5, - 0x3, 0x2, 0x2, 0x2, 0xe7, 0x4fc, 0x3, 0x2, 0x2, 0x2, 0xe9, 0x4ff, 0x3, - 0x2, 0x2, 0x2, 0xeb, 0x508, 0x3, 0x2, 0x2, 0x2, 0xed, 0x50b, 0x3, 0x2, - 0x2, 0x2, 0xef, 0x511, 0x3, 0x2, 0x2, 0x2, 0xf1, 0x517, 0x3, 0x2, 0x2, - 0x2, 0xf3, 0x51f, 0x3, 0x2, 0x2, 0x2, 0xf5, 0x529, 0x3, 0x2, 0x2, 0x2, - 0xf7, 0x532, 0x3, 0x2, 0x2, 0x2, 0xf9, 0x53b, 0x3, 0x2, 0x2, 0x2, 0xfb, - 0x543, 0x3, 0x2, 0x2, 0x2, 0xfd, 0x54e, 0x3, 0x2, 0x2, 0x2, 0xff, 0x556, - 0x3, 0x2, 0x2, 0x2, 0x101, 0x55c, 0x3, 0x2, 0x2, 0x2, 0x103, 0x563, - 0x3, 0x2, 0x2, 0x2, 0x105, 0x56a, 0x3, 0x2, 0x2, 0x2, 0x107, 0x571, - 0x3, 0x2, 0x2, 0x2, 0x109, 0x579, 0x3, 0x2, 0x2, 0x2, 0x10b, 0x581, - 0x3, 0x2, 0x2, 0x2, 0x10d, 0x58c, 0x3, 0x2, 0x2, 0x2, 0x10f, 0x592, - 0x3, 0x2, 0x2, 0x2, 0x111, 0x599, 0x3, 0x2, 0x2, 0x2, 0x113, 0x5a0, - 0x3, 0x2, 0x2, 0x2, 0x115, 0x5a7, 0x3, 0x2, 0x2, 0x2, 0x117, 0x5ae, - 0x3, 0x2, 0x2, 0x2, 0x119, 0x5b3, 0x3, 0x2, 0x2, 0x2, 0x11b, 0x5b9, - 0x3, 0x2, 0x2, 0x2, 0x11d, 0x5bd, 0x3, 0x2, 0x2, 0x2, 0x11f, 0x5c6, - 0x3, 0x2, 0x2, 0x2, 0x121, 0x5cb, 0x3, 0x2, 0x2, 0x2, 0x123, 0x5d2, - 0x3, 0x2, 0x2, 0x2, 0x125, 0x5d8, 0x3, 0x2, 0x2, 0x2, 0x127, 0x5dd, - 0x3, 0x2, 0x2, 0x2, 0x129, 0x5e7, 0x3, 0x2, 0x2, 0x2, 0x12b, 0x5ec, - 0x3, 0x2, 0x2, 0x2, 0x12d, 0x5f3, 0x3, 0x2, 0x2, 0x2, 0x12f, 0x5fa, - 0x3, 0x2, 0x2, 0x2, 0x131, 0x600, 0x3, 0x2, 0x2, 0x2, 0x133, 0x607, - 0x3, 0x2, 0x2, 0x2, 0x135, 0x611, 0x3, 0x2, 0x2, 0x2, 0x137, 0x616, - 0x3, 0x2, 0x2, 0x2, 0x139, 0x61b, 0x3, 0x2, 0x2, 0x2, 0x13b, 0x620, - 0x3, 0x2, 0x2, 0x2, 0x13d, 0x628, 0x3, 0x2, 0x2, 0x2, 0x13f, 0x632, - 0x3, 0x2, 0x2, 0x2, 0x141, 0x635, 0x3, 0x2, 0x2, 0x2, 0x143, 0x639, - 0x3, 0x2, 0x2, 0x2, 0x145, 0x640, 0x3, 0x2, 0x2, 0x2, 0x147, 0x649, - 0x3, 0x2, 0x2, 0x2, 0x149, 0x64e, 0x3, 0x2, 0x2, 0x2, 0x14b, 0x657, - 0x3, 0x2, 0x2, 0x2, 0x14d, 0x65b, 0x3, 0x2, 0x2, 0x2, 0x14f, 0x660, - 0x3, 0x2, 0x2, 0x2, 0x151, 0x666, 0x3, 0x2, 0x2, 0x2, 0x153, 0x66d, - 0x3, 0x2, 0x2, 0x2, 0x155, 0x671, 0x3, 0x2, 0x2, 0x2, 0x157, 0x677, - 0x3, 0x2, 0x2, 0x2, 0x159, 0x67c, 0x3, 0x2, 0x2, 0x2, 0x15b, 0x683, - 0x3, 0x2, 0x2, 0x2, 0x15d, 0x688, 0x3, 0x2, 0x2, 0x2, 0x15f, 0x68f, - 0x3, 0x2, 0x2, 0x2, 0x161, 0x695, 0x3, 0x2, 0x2, 0x2, 0x163, 0x69a, - 0x3, 0x2, 0x2, 0x2, 0x165, 0x69f, 0x3, 0x2, 0x2, 0x2, 0x167, 0x6a5, - 0x3, 0x2, 0x2, 0x2, 0x169, 0x6b4, 0x3, 0x2, 0x2, 0x2, 0x16b, 0x6b6, - 0x3, 0x2, 0x2, 0x2, 0x16d, 0x6bc, 0x3, 0x2, 0x2, 0x2, 0x16f, 0x6eb, - 0x3, 0x2, 0x2, 0x2, 0x171, 0x739, 0x3, 0x2, 0x2, 0x2, 0x173, 0x73b, - 0x3, 0x2, 0x2, 0x2, 0x175, 0x742, 0x3, 0x2, 0x2, 0x2, 0x177, 0x746, - 0x3, 0x2, 0x2, 0x2, 0x179, 0x74d, 0x3, 0x2, 0x2, 0x2, 0x17b, 0x75c, - 0x3, 0x2, 0x2, 0x2, 0x17d, 0x75e, 0x3, 0x2, 0x2, 0x2, 0x17f, 0x760, - 0x3, 0x2, 0x2, 0x2, 0x181, 0x762, 0x3, 0x2, 0x2, 0x2, 0x183, 0x764, - 0x3, 0x2, 0x2, 0x2, 0x185, 0x766, 0x3, 0x2, 0x2, 0x2, 0x187, 0x768, - 0x3, 0x2, 0x2, 0x2, 0x189, 0x76a, 0x3, 0x2, 0x2, 0x2, 0x18b, 0x76c, - 0x3, 0x2, 0x2, 0x2, 0x18d, 0x76e, 0x3, 0x2, 0x2, 0x2, 0x18f, 0x770, - 0x3, 0x2, 0x2, 0x2, 0x191, 0x772, 0x3, 0x2, 0x2, 0x2, 0x193, 0x774, - 0x3, 0x2, 0x2, 0x2, 0x195, 0x776, 0x3, 0x2, 0x2, 0x2, 0x197, 0x778, - 0x3, 0x2, 0x2, 0x2, 0x199, 0x77a, 0x3, 0x2, 0x2, 0x2, 0x19b, 0x77c, - 0x3, 0x2, 0x2, 0x2, 0x19d, 0x77e, 0x3, 0x2, 0x2, 0x2, 0x19f, 0x780, - 0x3, 0x2, 0x2, 0x2, 0x1a1, 0x782, 0x3, 0x2, 0x2, 0x2, 0x1a3, 0x784, - 0x3, 0x2, 0x2, 0x2, 0x1a5, 0x786, 0x3, 0x2, 0x2, 0x2, 0x1a7, 0x788, - 0x3, 0x2, 0x2, 0x2, 0x1a9, 0x78a, 0x3, 0x2, 0x2, 0x2, 0x1ab, 0x78c, - 0x3, 0x2, 0x2, 0x2, 0x1ad, 0x78e, 0x3, 0x2, 0x2, 0x2, 0x1af, 0x790, - 0x3, 0x2, 0x2, 0x2, 0x1b1, 0x792, 0x3, 0x2, 0x2, 0x2, 0x1b3, 0x794, - 0x3, 0x2, 0x2, 0x2, 0x1b5, 0x796, 0x3, 0x2, 0x2, 0x2, 0x1b7, 0x798, - 0x3, 0x2, 0x2, 0x2, 0x1b9, 0x79b, 0x3, 0x2, 0x2, 0x2, 0x1bb, 0x79d, - 0x3, 0x2, 0x2, 0x2, 0x1bd, 0x79f, 0x3, 0x2, 0x2, 0x2, 0x1bf, 0x7a1, - 0x3, 0x2, 0x2, 0x2, 0x1c1, 0x7a3, 0x3, 0x2, 0x2, 0x2, 0x1c3, 0x7a5, - 0x3, 0x2, 0x2, 0x2, 0x1c5, 0x7a8, 0x3, 0x2, 0x2, 0x2, 0x1c7, 0x7aa, - 0x3, 0x2, 0x2, 0x2, 0x1c9, 0x7ac, 0x3, 0x2, 0x2, 0x2, 0x1cb, 0x7af, - 0x3, 0x2, 0x2, 0x2, 0x1cd, 0x7b1, 0x3, 0x2, 0x2, 0x2, 0x1cf, 0x7b4, - 0x3, 0x2, 0x2, 0x2, 0x1d1, 0x7b6, 0x3, 0x2, 0x2, 0x2, 0x1d3, 0x7b8, - 0x3, 0x2, 0x2, 0x2, 0x1d5, 0x7ba, 0x3, 0x2, 0x2, 0x2, 0x1d7, 0x7bd, - 0x3, 0x2, 0x2, 0x2, 0x1d9, 0x7bf, 0x3, 0x2, 0x2, 0x2, 0x1db, 0x7c5, - 0x3, 0x2, 0x2, 0x2, 0x1dd, 0x7c7, 0x3, 0x2, 0x2, 0x2, 0x1df, 0x7c9, - 0x3, 0x2, 0x2, 0x2, 0x1e1, 0x7cb, 0x3, 0x2, 0x2, 0x2, 0x1e3, 0x7cd, - 0x3, 0x2, 0x2, 0x2, 0x1e5, 0x7cf, 0x3, 0x2, 0x2, 0x2, 0x1e7, 0x7d1, - 0x3, 0x2, 0x2, 0x2, 0x1e9, 0x7d3, 0x3, 0x2, 0x2, 0x2, 0x1eb, 0x7d5, - 0x3, 0x2, 0x2, 0x2, 0x1ed, 0x7d7, 0x3, 0x2, 0x2, 0x2, 0x1ef, 0x7d9, - 0x3, 0x2, 0x2, 0x2, 0x1f1, 0x7db, 0x3, 0x2, 0x2, 0x2, 0x1f3, 0x7dd, - 0x3, 0x2, 0x2, 0x2, 0x1f5, 0x7eb, 0x3, 0x2, 0x2, 0x2, 0x1f7, 0x7f9, - 0x3, 0x2, 0x2, 0x2, 0x1f9, 0x1fa, 0x5, 0x17b, 0xbe, 0x2, 0x1fa, 0x1fb, - 0x5, 0x181, 0xc1, 0x2, 0x1fb, 0x1fc, 0x5, 0x181, 0xc1, 0x2, 0x1fc, 0x4, - 0x3, 0x2, 0x2, 0x2, 0x1fd, 0x1fe, 0x5, 0x17b, 0xbe, 0x2, 0x1fe, 0x1ff, - 0x5, 0x185, 0xc3, 0x2, 0x1ff, 0x200, 0x5, 0x1a1, 0xd1, 0x2, 0x200, 0x201, - 0x5, 0x183, 0xc2, 0x2, 0x201, 0x202, 0x5, 0x19d, 0xcf, 0x2, 0x202, 0x6, - 0x3, 0x2, 0x2, 0x2, 0x203, 0x204, 0x5, 0x17b, 0xbe, 0x2, 0x204, 0x205, - 0x5, 0x191, 0xc9, 0x2, 0x205, 0x206, 0x5, 0x18b, 0xc6, 0x2, 0x206, 0x207, - 0x5, 0x17b, 0xbe, 0x2, 0x207, 0x208, 0x5, 0x19f, 0xd0, 0x2, 0x208, 0x8, - 0x3, 0x2, 0x2, 0x2, 0x209, 0x20a, 0x5, 0x17b, 0xbe, 0x2, 0x20a, 0x20b, - 0x5, 0x191, 0xc9, 0x2, 0x20b, 0x20c, 0x5, 0x191, 0xc9, 0x2, 0x20c, 0xa, - 0x3, 0x2, 0x2, 0x2, 0x20d, 0x20e, 0x5, 0x17b, 0xbe, 0x2, 0x20e, 0x20f, - 0x5, 0x191, 0xc9, 0x2, 0x20f, 0x210, 0x5, 0x1a1, 0xd1, 0x2, 0x210, 0x211, - 0x5, 0x183, 0xc2, 0x2, 0x211, 0x212, 0x5, 0x19d, 0xcf, 0x2, 0x212, 0xc, - 0x3, 0x2, 0x2, 0x2, 0x213, 0x214, 0x5, 0x17b, 0xbe, 0x2, 0x214, 0x215, - 0x5, 0x195, 0xcb, 0x2, 0x215, 0x216, 0x5, 0x181, 0xc1, 0x2, 0x216, 0xe, - 0x3, 0x2, 0x2, 0x2, 0x217, 0x218, 0x5, 0x17b, 0xbe, 0x2, 0x218, 0x219, - 0x5, 0x195, 0xcb, 0x2, 0x219, 0x21a, 0x5, 0x1a1, 0xd1, 0x2, 0x21a, 0x21b, - 0x5, 0x18b, 0xc6, 0x2, 0x21b, 0x10, 0x3, 0x2, 0x2, 0x2, 0x21c, 0x21d, - 0x5, 0x17b, 0xbe, 0x2, 0x21d, 0x21e, 0x5, 0x195, 0xcb, 0x2, 0x21e, 0x21f, - 0x5, 0x1ab, 0xd6, 0x2, 0x21f, 0x12, 0x3, 0x2, 0x2, 0x2, 0x220, 0x221, - 0x5, 0x17b, 0xbe, 0x2, 0x221, 0x222, 0x5, 0x19d, 0xcf, 0x2, 0x222, 0x223, - 0x5, 0x19d, 0xcf, 0x2, 0x223, 0x224, 0x5, 0x17b, 0xbe, 0x2, 0x224, 0x225, - 0x5, 0x1ab, 0xd6, 0x2, 0x225, 0x14, 0x3, 0x2, 0x2, 0x2, 0x226, 0x227, - 0x5, 0x17b, 0xbe, 0x2, 0x227, 0x228, 0x5, 0x19f, 0xd0, 0x2, 0x228, 0x16, - 0x3, 0x2, 0x2, 0x2, 0x229, 0x22a, 0x5, 0x17b, 0xbe, 0x2, 0x22a, 0x22b, - 0x5, 0x19f, 0xd0, 0x2, 0x22b, 0x22c, 0x5, 0x17f, 0xc0, 0x2, 0x22c, 0x238, - 0x3, 0x2, 0x2, 0x2, 0x22d, 0x22e, 0x5, 0x17b, 0xbe, 0x2, 0x22e, 0x22f, - 0x5, 0x19f, 0xd0, 0x2, 0x22f, 0x230, 0x5, 0x17f, 0xc0, 0x2, 0x230, 0x231, - 0x5, 0x183, 0xc2, 0x2, 0x231, 0x232, 0x5, 0x195, 0xcb, 0x2, 0x232, 0x233, - 0x5, 0x181, 0xc1, 0x2, 0x233, 0x234, 0x5, 0x18b, 0xc6, 0x2, 0x234, 0x235, - 0x5, 0x195, 0xcb, 0x2, 0x235, 0x236, 0x5, 0x187, 0xc4, 0x2, 0x236, 0x238, - 0x3, 0x2, 0x2, 0x2, 0x237, 0x229, 0x3, 0x2, 0x2, 0x2, 0x237, 0x22d, - 0x3, 0x2, 0x2, 0x2, 0x238, 0x18, 0x3, 0x2, 0x2, 0x2, 0x239, 0x23a, 0x5, - 0x17b, 0xbe, 0x2, 0x23a, 0x23b, 0x5, 0x19f, 0xd0, 0x2, 0x23b, 0x23c, - 0x5, 0x197, 0xcc, 0x2, 0x23c, 0x23d, 0x5, 0x185, 0xc3, 0x2, 0x23d, 0x1a, - 0x3, 0x2, 0x2, 0x2, 0x23e, 0x23f, 0x5, 0x17b, 0xbe, 0x2, 0x23f, 0x240, - 0x5, 0x19f, 0xd0, 0x2, 0x240, 0x241, 0x5, 0x1ab, 0xd6, 0x2, 0x241, 0x242, - 0x5, 0x195, 0xcb, 0x2, 0x242, 0x243, 0x5, 0x17f, 0xc0, 0x2, 0x243, 0x1c, - 0x3, 0x2, 0x2, 0x2, 0x244, 0x245, 0x5, 0x17b, 0xbe, 0x2, 0x245, 0x246, - 0x5, 0x1a1, 0xd1, 0x2, 0x246, 0x247, 0x5, 0x1a1, 0xd1, 0x2, 0x247, 0x248, - 0x5, 0x17b, 0xbe, 0x2, 0x248, 0x249, 0x5, 0x17f, 0xc0, 0x2, 0x249, 0x24a, - 0x5, 0x189, 0xc5, 0x2, 0x24a, 0x1e, 0x3, 0x2, 0x2, 0x2, 0x24b, 0x24c, - 0x5, 0x17d, 0xbf, 0x2, 0x24c, 0x24d, 0x5, 0x183, 0xc2, 0x2, 0x24d, 0x24e, - 0x5, 0x1a1, 0xd1, 0x2, 0x24e, 0x24f, 0x5, 0x1a7, 0xd4, 0x2, 0x24f, 0x250, - 0x5, 0x183, 0xc2, 0x2, 0x250, 0x251, 0x5, 0x183, 0xc2, 0x2, 0x251, 0x252, - 0x5, 0x195, 0xcb, 0x2, 0x252, 0x20, 0x3, 0x2, 0x2, 0x2, 0x253, 0x254, - 0x5, 0x17d, 0xbf, 0x2, 0x254, 0x255, 0x5, 0x197, 0xcc, 0x2, 0x255, 0x256, - 0x5, 0x1a1, 0xd1, 0x2, 0x256, 0x257, 0x5, 0x189, 0xc5, 0x2, 0x257, 0x22, - 0x3, 0x2, 0x2, 0x2, 0x258, 0x259, 0x5, 0x17d, 0xbf, 0x2, 0x259, 0x25a, - 0x5, 0x1ab, 0xd6, 0x2, 0x25a, 0x24, 0x3, 0x2, 0x2, 0x2, 0x25b, 0x25c, - 0x5, 0x17f, 0xc0, 0x2, 0x25c, 0x25d, 0x5, 0x17b, 0xbe, 0x2, 0x25d, 0x25e, - 0x5, 0x19f, 0xd0, 0x2, 0x25e, 0x25f, 0x5, 0x183, 0xc2, 0x2, 0x25f, 0x26, - 0x3, 0x2, 0x2, 0x2, 0x260, 0x261, 0x5, 0x17f, 0xc0, 0x2, 0x261, 0x262, - 0x5, 0x17b, 0xbe, 0x2, 0x262, 0x263, 0x5, 0x19f, 0xd0, 0x2, 0x263, 0x264, - 0x5, 0x1a1, 0xd1, 0x2, 0x264, 0x28, 0x3, 0x2, 0x2, 0x2, 0x265, 0x266, - 0x5, 0x17f, 0xc0, 0x2, 0x266, 0x267, 0x5, 0x189, 0xc5, 0x2, 0x267, 0x268, - 0x5, 0x183, 0xc2, 0x2, 0x268, 0x269, 0x5, 0x17f, 0xc0, 0x2, 0x269, 0x26a, - 0x5, 0x18f, 0xc8, 0x2, 0x26a, 0x2a, 0x3, 0x2, 0x2, 0x2, 0x26b, 0x26c, - 0x5, 0x17f, 0xc0, 0x2, 0x26c, 0x26d, 0x5, 0x191, 0xc9, 0x2, 0x26d, 0x26e, - 0x5, 0x183, 0xc2, 0x2, 0x26e, 0x26f, 0x5, 0x17b, 0xbe, 0x2, 0x26f, 0x270, - 0x5, 0x19d, 0xcf, 0x2, 0x270, 0x2c, 0x3, 0x2, 0x2, 0x2, 0x271, 0x272, - 0x5, 0x17f, 0xc0, 0x2, 0x272, 0x273, 0x5, 0x191, 0xc9, 0x2, 0x273, 0x274, - 0x5, 0x1a3, 0xd2, 0x2, 0x274, 0x275, 0x5, 0x19f, 0xd0, 0x2, 0x275, 0x276, - 0x5, 0x1a1, 0xd1, 0x2, 0x276, 0x277, 0x5, 0x183, 0xc2, 0x2, 0x277, 0x278, - 0x5, 0x19d, 0xcf, 0x2, 0x278, 0x2e, 0x3, 0x2, 0x2, 0x2, 0x279, 0x27a, - 0x5, 0x17f, 0xc0, 0x2, 0x27a, 0x27b, 0x5, 0x197, 0xcc, 0x2, 0x27b, 0x27c, - 0x5, 0x181, 0xc1, 0x2, 0x27c, 0x27d, 0x5, 0x183, 0xc2, 0x2, 0x27d, 0x27e, - 0x5, 0x17f, 0xc0, 0x2, 0x27e, 0x30, 0x3, 0x2, 0x2, 0x2, 0x27f, 0x280, - 0x5, 0x17f, 0xc0, 0x2, 0x280, 0x281, 0x5, 0x197, 0xcc, 0x2, 0x281, 0x282, - 0x5, 0x191, 0xc9, 0x2, 0x282, 0x283, 0x5, 0x191, 0xc9, 0x2, 0x283, 0x284, - 0x5, 0x17b, 0xbe, 0x2, 0x284, 0x285, 0x5, 0x1a1, 0xd1, 0x2, 0x285, 0x286, - 0x5, 0x183, 0xc2, 0x2, 0x286, 0x32, 0x3, 0x2, 0x2, 0x2, 0x287, 0x288, - 0x5, 0x17f, 0xc0, 0x2, 0x288, 0x289, 0x5, 0x197, 0xcc, 0x2, 0x289, 0x28a, - 0x5, 0x191, 0xc9, 0x2, 0x28a, 0x28b, 0x5, 0x1a3, 0xd2, 0x2, 0x28b, 0x28c, - 0x5, 0x193, 0xca, 0x2, 0x28c, 0x28d, 0x5, 0x195, 0xcb, 0x2, 0x28d, 0x34, - 0x3, 0x2, 0x2, 0x2, 0x28e, 0x28f, 0x5, 0x17f, 0xc0, 0x2, 0x28f, 0x290, - 0x5, 0x197, 0xcc, 0x2, 0x290, 0x291, 0x5, 0x193, 0xca, 0x2, 0x291, 0x292, - 0x5, 0x193, 0xca, 0x2, 0x292, 0x293, 0x5, 0x183, 0xc2, 0x2, 0x293, 0x294, - 0x5, 0x195, 0xcb, 0x2, 0x294, 0x295, 0x5, 0x1a1, 0xd1, 0x2, 0x295, 0x36, - 0x3, 0x2, 0x2, 0x2, 0x296, 0x297, 0x5, 0x17f, 0xc0, 0x2, 0x297, 0x298, - 0x5, 0x197, 0xcc, 0x2, 0x298, 0x299, 0x5, 0x195, 0xcb, 0x2, 0x299, 0x29a, - 0x5, 0x19f, 0xd0, 0x2, 0x29a, 0x29b, 0x5, 0x1a1, 0xd1, 0x2, 0x29b, 0x29c, - 0x5, 0x19d, 0xcf, 0x2, 0x29c, 0x29d, 0x5, 0x17b, 0xbe, 0x2, 0x29d, 0x29e, - 0x5, 0x18b, 0xc6, 0x2, 0x29e, 0x29f, 0x5, 0x195, 0xcb, 0x2, 0x29f, 0x2a0, - 0x5, 0x1a1, 0xd1, 0x2, 0x2a0, 0x38, 0x3, 0x2, 0x2, 0x2, 0x2a1, 0x2a2, - 0x5, 0x17f, 0xc0, 0x2, 0x2a2, 0x2a3, 0x5, 0x19d, 0xcf, 0x2, 0x2a3, 0x2a4, - 0x5, 0x183, 0xc2, 0x2, 0x2a4, 0x2a5, 0x5, 0x17b, 0xbe, 0x2, 0x2a5, 0x2a6, - 0x5, 0x1a1, 0xd1, 0x2, 0x2a6, 0x2a7, 0x5, 0x183, 0xc2, 0x2, 0x2a7, 0x3a, - 0x3, 0x2, 0x2, 0x2, 0x2a8, 0x2a9, 0x5, 0x17f, 0xc0, 0x2, 0x2a9, 0x2aa, - 0x5, 0x19d, 0xcf, 0x2, 0x2aa, 0x2ab, 0x5, 0x197, 0xcc, 0x2, 0x2ab, 0x2ac, - 0x5, 0x19f, 0xd0, 0x2, 0x2ac, 0x2ad, 0x5, 0x19f, 0xd0, 0x2, 0x2ad, 0x3c, - 0x3, 0x2, 0x2, 0x2, 0x2ae, 0x2af, 0x5, 0x17f, 0xc0, 0x2, 0x2af, 0x2b0, - 0x5, 0x1a3, 0xd2, 0x2, 0x2b0, 0x2b1, 0x5, 0x17d, 0xbf, 0x2, 0x2b1, 0x2b2, - 0x5, 0x183, 0xc2, 0x2, 0x2b2, 0x3e, 0x3, 0x2, 0x2, 0x2, 0x2b3, 0x2b4, - 0x5, 0x181, 0xc1, 0x2, 0x2b4, 0x2b5, 0x5, 0x17b, 0xbe, 0x2, 0x2b5, 0x2b6, - 0x5, 0x1a1, 0xd1, 0x2, 0x2b6, 0x2b7, 0x5, 0x17b, 0xbe, 0x2, 0x2b7, 0x2b8, - 0x5, 0x17d, 0xbf, 0x2, 0x2b8, 0x2b9, 0x5, 0x17b, 0xbe, 0x2, 0x2b9, 0x2ba, - 0x5, 0x19f, 0xd0, 0x2, 0x2ba, 0x2bb, 0x5, 0x183, 0xc2, 0x2, 0x2bb, 0x40, - 0x3, 0x2, 0x2, 0x2, 0x2bc, 0x2bd, 0x5, 0x181, 0xc1, 0x2, 0x2bd, 0x2be, - 0x5, 0x17b, 0xbe, 0x2, 0x2be, 0x2bf, 0x5, 0x1a1, 0xd1, 0x2, 0x2bf, 0x2c0, - 0x5, 0x17b, 0xbe, 0x2, 0x2c0, 0x2c1, 0x5, 0x17d, 0xbf, 0x2, 0x2c1, 0x2c2, - 0x5, 0x17b, 0xbe, 0x2, 0x2c2, 0x2c3, 0x5, 0x19f, 0xd0, 0x2, 0x2c3, 0x2c4, - 0x5, 0x183, 0xc2, 0x2, 0x2c4, 0x2c5, 0x5, 0x19f, 0xd0, 0x2, 0x2c5, 0x42, - 0x3, 0x2, 0x2, 0x2, 0x2c6, 0x2c7, 0x5, 0x181, 0xc1, 0x2, 0x2c7, 0x2c8, - 0x5, 0x17b, 0xbe, 0x2, 0x2c8, 0x2c9, 0x5, 0x1a1, 0xd1, 0x2, 0x2c9, 0x2ca, - 0x5, 0x183, 0xc2, 0x2, 0x2ca, 0x44, 0x3, 0x2, 0x2, 0x2, 0x2cb, 0x2cc, - 0x5, 0x181, 0xc1, 0x2, 0x2cc, 0x2cd, 0x5, 0x17b, 0xbe, 0x2, 0x2cd, 0x2ce, - 0x5, 0x1ab, 0xd6, 0x2, 0x2ce, 0x46, 0x3, 0x2, 0x2, 0x2, 0x2cf, 0x2d0, - 0x5, 0x181, 0xc1, 0x2, 0x2d0, 0x2d1, 0x5, 0x183, 0xc2, 0x2, 0x2d1, 0x2d2, - 0x5, 0x181, 0xc1, 0x2, 0x2d2, 0x2d3, 0x5, 0x1a3, 0xd2, 0x2, 0x2d3, 0x2d4, - 0x5, 0x199, 0xcd, 0x2, 0x2d4, 0x2d5, 0x5, 0x191, 0xc9, 0x2, 0x2d5, 0x2d6, - 0x5, 0x18b, 0xc6, 0x2, 0x2d6, 0x2d7, 0x5, 0x17f, 0xc0, 0x2, 0x2d7, 0x2d8, - 0x5, 0x17b, 0xbe, 0x2, 0x2d8, 0x2d9, 0x5, 0x1a1, 0xd1, 0x2, 0x2d9, 0x2da, - 0x5, 0x183, 0xc2, 0x2, 0x2da, 0x48, 0x3, 0x2, 0x2, 0x2, 0x2db, 0x2dc, - 0x5, 0x181, 0xc1, 0x2, 0x2dc, 0x2dd, 0x5, 0x183, 0xc2, 0x2, 0x2dd, 0x2de, - 0x5, 0x185, 0xc3, 0x2, 0x2de, 0x2df, 0x5, 0x17b, 0xbe, 0x2, 0x2df, 0x2e0, - 0x5, 0x1a3, 0xd2, 0x2, 0x2e0, 0x2e1, 0x5, 0x191, 0xc9, 0x2, 0x2e1, 0x2e2, - 0x5, 0x1a1, 0xd1, 0x2, 0x2e2, 0x4a, 0x3, 0x2, 0x2, 0x2, 0x2e3, 0x2e4, - 0x5, 0x181, 0xc1, 0x2, 0x2e4, 0x2e5, 0x5, 0x183, 0xc2, 0x2, 0x2e5, 0x2e6, - 0x5, 0x191, 0xc9, 0x2, 0x2e6, 0x2e7, 0x5, 0x17b, 0xbe, 0x2, 0x2e7, 0x2e8, - 0x5, 0x1ab, 0xd6, 0x2, 0x2e8, 0x4c, 0x3, 0x2, 0x2, 0x2, 0x2e9, 0x2ea, - 0x5, 0x181, 0xc1, 0x2, 0x2ea, 0x2eb, 0x5, 0x183, 0xc2, 0x2, 0x2eb, 0x2ec, - 0x5, 0x191, 0xc9, 0x2, 0x2ec, 0x2ed, 0x5, 0x183, 0xc2, 0x2, 0x2ed, 0x2ee, - 0x5, 0x1a1, 0xd1, 0x2, 0x2ee, 0x2ef, 0x5, 0x183, 0xc2, 0x2, 0x2ef, 0x4e, - 0x3, 0x2, 0x2, 0x2, 0x2f0, 0x2f1, 0x5, 0x181, 0xc1, 0x2, 0x2f1, 0x2f2, - 0x5, 0x183, 0xc2, 0x2, 0x2f2, 0x2f3, 0x5, 0x19f, 0xd0, 0x2, 0x2f3, 0x2f4, - 0x5, 0x17f, 0xc0, 0x2, 0x2f4, 0x50, 0x3, 0x2, 0x2, 0x2, 0x2f5, 0x2f6, - 0x5, 0x181, 0xc1, 0x2, 0x2f6, 0x2f7, 0x5, 0x183, 0xc2, 0x2, 0x2f7, 0x2f8, - 0x5, 0x19f, 0xd0, 0x2, 0x2f8, 0x2f9, 0x5, 0x17f, 0xc0, 0x2, 0x2f9, 0x2fa, - 0x5, 0x183, 0xc2, 0x2, 0x2fa, 0x2fb, 0x5, 0x195, 0xcb, 0x2, 0x2fb, 0x2fc, - 0x5, 0x181, 0xc1, 0x2, 0x2fc, 0x2fd, 0x5, 0x18b, 0xc6, 0x2, 0x2fd, 0x2fe, - 0x5, 0x195, 0xcb, 0x2, 0x2fe, 0x2ff, 0x5, 0x187, 0xc4, 0x2, 0x2ff, 0x52, - 0x3, 0x2, 0x2, 0x2, 0x300, 0x301, 0x5, 0x181, 0xc1, 0x2, 0x301, 0x302, - 0x5, 0x183, 0xc2, 0x2, 0x302, 0x303, 0x5, 0x19f, 0xd0, 0x2, 0x303, 0x304, - 0x5, 0x17f, 0xc0, 0x2, 0x304, 0x305, 0x5, 0x19d, 0xcf, 0x2, 0x305, 0x306, - 0x5, 0x18b, 0xc6, 0x2, 0x306, 0x307, 0x5, 0x17d, 0xbf, 0x2, 0x307, 0x308, - 0x5, 0x183, 0xc2, 0x2, 0x308, 0x54, 0x3, 0x2, 0x2, 0x2, 0x309, 0x30a, - 0x5, 0x181, 0xc1, 0x2, 0x30a, 0x30b, 0x5, 0x183, 0xc2, 0x2, 0x30b, 0x30c, - 0x5, 0x1a1, 0xd1, 0x2, 0x30c, 0x30d, 0x5, 0x17b, 0xbe, 0x2, 0x30d, 0x30e, - 0x5, 0x17f, 0xc0, 0x2, 0x30e, 0x30f, 0x5, 0x189, 0xc5, 0x2, 0x30f, 0x56, - 0x3, 0x2, 0x2, 0x2, 0x310, 0x311, 0x5, 0x181, 0xc1, 0x2, 0x311, 0x312, - 0x5, 0x18b, 0xc6, 0x2, 0x312, 0x313, 0x5, 0x17f, 0xc0, 0x2, 0x313, 0x314, - 0x5, 0x1a1, 0xd1, 0x2, 0x314, 0x315, 0x5, 0x18b, 0xc6, 0x2, 0x315, 0x316, - 0x5, 0x197, 0xcc, 0x2, 0x316, 0x317, 0x5, 0x195, 0xcb, 0x2, 0x317, 0x318, - 0x5, 0x17b, 0xbe, 0x2, 0x318, 0x319, 0x5, 0x19d, 0xcf, 0x2, 0x319, 0x31a, - 0x5, 0x18b, 0xc6, 0x2, 0x31a, 0x31b, 0x5, 0x183, 0xc2, 0x2, 0x31b, 0x31c, - 0x5, 0x19f, 0xd0, 0x2, 0x31c, 0x58, 0x3, 0x2, 0x2, 0x2, 0x31d, 0x31e, - 0x5, 0x181, 0xc1, 0x2, 0x31e, 0x31f, 0x5, 0x18b, 0xc6, 0x2, 0x31f, 0x320, - 0x5, 0x17f, 0xc0, 0x2, 0x320, 0x321, 0x5, 0x1a1, 0xd1, 0x2, 0x321, 0x322, - 0x5, 0x18b, 0xc6, 0x2, 0x322, 0x323, 0x5, 0x197, 0xcc, 0x2, 0x323, 0x324, - 0x5, 0x195, 0xcb, 0x2, 0x324, 0x325, 0x5, 0x17b, 0xbe, 0x2, 0x325, 0x326, - 0x5, 0x19d, 0xcf, 0x2, 0x326, 0x327, 0x5, 0x1ab, 0xd6, 0x2, 0x327, 0x5a, - 0x3, 0x2, 0x2, 0x2, 0x328, 0x329, 0x5, 0x181, 0xc1, 0x2, 0x329, 0x32a, - 0x5, 0x18b, 0xc6, 0x2, 0x32a, 0x32b, 0x5, 0x19f, 0xd0, 0x2, 0x32b, 0x32c, - 0x5, 0x18f, 0xc8, 0x2, 0x32c, 0x5c, 0x3, 0x2, 0x2, 0x2, 0x32d, 0x32e, - 0x5, 0x181, 0xc1, 0x2, 0x32e, 0x32f, 0x5, 0x18b, 0xc6, 0x2, 0x32f, 0x330, - 0x5, 0x19f, 0xd0, 0x2, 0x330, 0x331, 0x5, 0x1a1, 0xd1, 0x2, 0x331, 0x332, - 0x5, 0x18b, 0xc6, 0x2, 0x332, 0x333, 0x5, 0x195, 0xcb, 0x2, 0x333, 0x334, - 0x5, 0x17f, 0xc0, 0x2, 0x334, 0x335, 0x5, 0x1a1, 0xd1, 0x2, 0x335, 0x5e, - 0x3, 0x2, 0x2, 0x2, 0x336, 0x337, 0x5, 0x181, 0xc1, 0x2, 0x337, 0x338, - 0x5, 0x18b, 0xc6, 0x2, 0x338, 0x339, 0x5, 0x19f, 0xd0, 0x2, 0x339, 0x33a, - 0x5, 0x1a1, 0xd1, 0x2, 0x33a, 0x33b, 0x5, 0x19d, 0xcf, 0x2, 0x33b, 0x33c, - 0x5, 0x18b, 0xc6, 0x2, 0x33c, 0x33d, 0x5, 0x17d, 0xbf, 0x2, 0x33d, 0x33e, - 0x5, 0x1a3, 0xd2, 0x2, 0x33e, 0x33f, 0x5, 0x1a1, 0xd1, 0x2, 0x33f, 0x340, - 0x5, 0x183, 0xc2, 0x2, 0x340, 0x341, 0x5, 0x181, 0xc1, 0x2, 0x341, 0x60, - 0x3, 0x2, 0x2, 0x2, 0x342, 0x343, 0x5, 0x181, 0xc1, 0x2, 0x343, 0x344, - 0x5, 0x19d, 0xcf, 0x2, 0x344, 0x345, 0x5, 0x197, 0xcc, 0x2, 0x345, 0x346, - 0x5, 0x199, 0xcd, 0x2, 0x346, 0x62, 0x3, 0x2, 0x2, 0x2, 0x347, 0x348, - 0x5, 0x183, 0xc2, 0x2, 0x348, 0x349, 0x5, 0x191, 0xc9, 0x2, 0x349, 0x34a, - 0x5, 0x19f, 0xd0, 0x2, 0x34a, 0x34b, 0x5, 0x183, 0xc2, 0x2, 0x34b, 0x64, - 0x3, 0x2, 0x2, 0x2, 0x34c, 0x34d, 0x5, 0x183, 0xc2, 0x2, 0x34d, 0x34e, - 0x5, 0x195, 0xcb, 0x2, 0x34e, 0x34f, 0x5, 0x181, 0xc1, 0x2, 0x34f, 0x66, - 0x3, 0x2, 0x2, 0x2, 0x350, 0x351, 0x5, 0x183, 0xc2, 0x2, 0x351, 0x352, - 0x5, 0x195, 0xcb, 0x2, 0x352, 0x353, 0x5, 0x187, 0xc4, 0x2, 0x353, 0x354, - 0x5, 0x18b, 0xc6, 0x2, 0x354, 0x355, 0x5, 0x195, 0xcb, 0x2, 0x355, 0x356, - 0x5, 0x183, 0xc2, 0x2, 0x356, 0x68, 0x3, 0x2, 0x2, 0x2, 0x357, 0x358, - 0x5, 0x183, 0xc2, 0x2, 0x358, 0x359, 0x5, 0x1a5, 0xd3, 0x2, 0x359, 0x35a, - 0x5, 0x183, 0xc2, 0x2, 0x35a, 0x35b, 0x5, 0x195, 0xcb, 0x2, 0x35b, 0x35c, - 0x5, 0x1a1, 0xd1, 0x2, 0x35c, 0x35d, 0x5, 0x19f, 0xd0, 0x2, 0x35d, 0x6a, - 0x3, 0x2, 0x2, 0x2, 0x35e, 0x35f, 0x5, 0x183, 0xc2, 0x2, 0x35f, 0x360, - 0x5, 0x1a9, 0xd5, 0x2, 0x360, 0x361, 0x5, 0x18b, 0xc6, 0x2, 0x361, 0x362, - 0x5, 0x19f, 0xd0, 0x2, 0x362, 0x363, 0x5, 0x1a1, 0xd1, 0x2, 0x363, 0x364, - 0x5, 0x19f, 0xd0, 0x2, 0x364, 0x6c, 0x3, 0x2, 0x2, 0x2, 0x365, 0x366, - 0x5, 0x183, 0xc2, 0x2, 0x366, 0x367, 0x5, 0x1a9, 0xd5, 0x2, 0x367, 0x368, - 0x5, 0x199, 0xcd, 0x2, 0x368, 0x369, 0x5, 0x191, 0xc9, 0x2, 0x369, 0x36a, - 0x5, 0x17b, 0xbe, 0x2, 0x36a, 0x36b, 0x5, 0x18b, 0xc6, 0x2, 0x36b, 0x36c, - 0x5, 0x195, 0xcb, 0x2, 0x36c, 0x6e, 0x3, 0x2, 0x2, 0x2, 0x36d, 0x36e, - 0x5, 0x183, 0xc2, 0x2, 0x36e, 0x36f, 0x5, 0x1a9, 0xd5, 0x2, 0x36f, 0x370, - 0x5, 0x199, 0xcd, 0x2, 0x370, 0x371, 0x5, 0x19d, 0xcf, 0x2, 0x371, 0x372, - 0x5, 0x183, 0xc2, 0x2, 0x372, 0x373, 0x5, 0x19f, 0xd0, 0x2, 0x373, 0x374, - 0x5, 0x19f, 0xd0, 0x2, 0x374, 0x375, 0x5, 0x18b, 0xc6, 0x2, 0x375, 0x376, - 0x5, 0x197, 0xcc, 0x2, 0x376, 0x377, 0x5, 0x195, 0xcb, 0x2, 0x377, 0x70, - 0x3, 0x2, 0x2, 0x2, 0x378, 0x379, 0x5, 0x183, 0xc2, 0x2, 0x379, 0x37a, - 0x5, 0x1a9, 0xd5, 0x2, 0x37a, 0x37b, 0x5, 0x1a1, 0xd1, 0x2, 0x37b, 0x37c, - 0x5, 0x19d, 0xcf, 0x2, 0x37c, 0x37d, 0x5, 0x17b, 0xbe, 0x2, 0x37d, 0x37e, - 0x5, 0x17f, 0xc0, 0x2, 0x37e, 0x37f, 0x5, 0x1a1, 0xd1, 0x2, 0x37f, 0x72, - 0x3, 0x2, 0x2, 0x2, 0x380, 0x381, 0x5, 0x185, 0xc3, 0x2, 0x381, 0x382, - 0x5, 0x183, 0xc2, 0x2, 0x382, 0x383, 0x5, 0x1a1, 0xd1, 0x2, 0x383, 0x384, - 0x5, 0x17f, 0xc0, 0x2, 0x384, 0x385, 0x5, 0x189, 0xc5, 0x2, 0x385, 0x386, - 0x5, 0x183, 0xc2, 0x2, 0x386, 0x387, 0x5, 0x19f, 0xd0, 0x2, 0x387, 0x74, - 0x3, 0x2, 0x2, 0x2, 0x388, 0x389, 0x5, 0x185, 0xc3, 0x2, 0x389, 0x38a, - 0x5, 0x18b, 0xc6, 0x2, 0x38a, 0x38b, 0x5, 0x195, 0xcb, 0x2, 0x38b, 0x38c, - 0x5, 0x17b, 0xbe, 0x2, 0x38c, 0x38d, 0x5, 0x191, 0xc9, 0x2, 0x38d, 0x76, - 0x3, 0x2, 0x2, 0x2, 0x38e, 0x38f, 0x5, 0x185, 0xc3, 0x2, 0x38f, 0x390, - 0x5, 0x18b, 0xc6, 0x2, 0x390, 0x391, 0x5, 0x19d, 0xcf, 0x2, 0x391, 0x392, - 0x5, 0x19f, 0xd0, 0x2, 0x392, 0x393, 0x5, 0x1a1, 0xd1, 0x2, 0x393, 0x78, - 0x3, 0x2, 0x2, 0x2, 0x394, 0x395, 0x5, 0x185, 0xc3, 0x2, 0x395, 0x396, - 0x5, 0x191, 0xc9, 0x2, 0x396, 0x397, 0x5, 0x1a3, 0xd2, 0x2, 0x397, 0x398, - 0x5, 0x19f, 0xd0, 0x2, 0x398, 0x399, 0x5, 0x189, 0xc5, 0x2, 0x399, 0x7a, - 0x3, 0x2, 0x2, 0x2, 0x39a, 0x39b, 0x5, 0x185, 0xc3, 0x2, 0x39b, 0x39c, - 0x5, 0x197, 0xcc, 0x2, 0x39c, 0x39d, 0x5, 0x19d, 0xcf, 0x2, 0x39d, 0x7c, - 0x3, 0x2, 0x2, 0x2, 0x39e, 0x39f, 0x5, 0x185, 0xc3, 0x2, 0x39f, 0x3a0, - 0x5, 0x197, 0xcc, 0x2, 0x3a0, 0x3a1, 0x5, 0x19d, 0xcf, 0x2, 0x3a1, 0x3a2, - 0x5, 0x193, 0xca, 0x2, 0x3a2, 0x3a3, 0x5, 0x17b, 0xbe, 0x2, 0x3a3, 0x3a4, - 0x5, 0x1a1, 0xd1, 0x2, 0x3a4, 0x7e, 0x3, 0x2, 0x2, 0x2, 0x3a5, 0x3a6, - 0x5, 0x185, 0xc3, 0x2, 0x3a6, 0x3a7, 0x5, 0x19d, 0xcf, 0x2, 0x3a7, 0x3a8, - 0x5, 0x183, 0xc2, 0x2, 0x3a8, 0x3a9, 0x5, 0x183, 0xc2, 0x2, 0x3a9, 0x3aa, - 0x5, 0x1ad, 0xd7, 0x2, 0x3aa, 0x3ab, 0x5, 0x183, 0xc2, 0x2, 0x3ab, 0x80, - 0x3, 0x2, 0x2, 0x2, 0x3ac, 0x3ad, 0x5, 0x185, 0xc3, 0x2, 0x3ad, 0x3ae, - 0x5, 0x19d, 0xcf, 0x2, 0x3ae, 0x3af, 0x5, 0x197, 0xcc, 0x2, 0x3af, 0x3b0, - 0x5, 0x193, 0xca, 0x2, 0x3b0, 0x82, 0x3, 0x2, 0x2, 0x2, 0x3b1, 0x3b2, - 0x5, 0x185, 0xc3, 0x2, 0x3b2, 0x3b3, 0x5, 0x1a3, 0xd2, 0x2, 0x3b3, 0x3b4, - 0x5, 0x191, 0xc9, 0x2, 0x3b4, 0x3b5, 0x5, 0x191, 0xc9, 0x2, 0x3b5, 0x84, - 0x3, 0x2, 0x2, 0x2, 0x3b6, 0x3b7, 0x5, 0x185, 0xc3, 0x2, 0x3b7, 0x3b8, - 0x5, 0x1a3, 0xd2, 0x2, 0x3b8, 0x3b9, 0x5, 0x195, 0xcb, 0x2, 0x3b9, 0x3ba, - 0x5, 0x17f, 0xc0, 0x2, 0x3ba, 0x3bb, 0x5, 0x1a1, 0xd1, 0x2, 0x3bb, 0x3bc, - 0x5, 0x18b, 0xc6, 0x2, 0x3bc, 0x3bd, 0x5, 0x197, 0xcc, 0x2, 0x3bd, 0x3be, - 0x5, 0x195, 0xcb, 0x2, 0x3be, 0x86, 0x3, 0x2, 0x2, 0x2, 0x3bf, 0x3c0, - 0x5, 0x187, 0xc4, 0x2, 0x3c0, 0x3c1, 0x5, 0x191, 0xc9, 0x2, 0x3c1, 0x3c2, - 0x5, 0x197, 0xcc, 0x2, 0x3c2, 0x3c3, 0x5, 0x17d, 0xbf, 0x2, 0x3c3, 0x3c4, - 0x5, 0x17b, 0xbe, 0x2, 0x3c4, 0x3c5, 0x5, 0x191, 0xc9, 0x2, 0x3c5, 0x88, - 0x3, 0x2, 0x2, 0x2, 0x3c6, 0x3c7, 0x5, 0x187, 0xc4, 0x2, 0x3c7, 0x3c8, - 0x5, 0x19d, 0xcf, 0x2, 0x3c8, 0x3c9, 0x5, 0x17b, 0xbe, 0x2, 0x3c9, 0x3ca, - 0x5, 0x195, 0xcb, 0x2, 0x3ca, 0x3cb, 0x5, 0x1a3, 0xd2, 0x2, 0x3cb, 0x3cc, - 0x5, 0x191, 0xc9, 0x2, 0x3cc, 0x3cd, 0x5, 0x17b, 0xbe, 0x2, 0x3cd, 0x3ce, - 0x5, 0x19d, 0xcf, 0x2, 0x3ce, 0x3cf, 0x5, 0x18b, 0xc6, 0x2, 0x3cf, 0x3d0, - 0x5, 0x1a1, 0xd1, 0x2, 0x3d0, 0x3d1, 0x5, 0x1ab, 0xd6, 0x2, 0x3d1, 0x8a, - 0x3, 0x2, 0x2, 0x2, 0x3d2, 0x3d3, 0x5, 0x187, 0xc4, 0x2, 0x3d3, 0x3d4, - 0x5, 0x19d, 0xcf, 0x2, 0x3d4, 0x3d5, 0x5, 0x197, 0xcc, 0x2, 0x3d5, 0x3d6, - 0x5, 0x1a3, 0xd2, 0x2, 0x3d6, 0x3d7, 0x5, 0x199, 0xcd, 0x2, 0x3d7, 0x8c, + 0x2, 0x2, 0x2, 0x2, 0x1f7, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1f9, 0x3, 0x2, + 0x2, 0x2, 0x3, 0x1fb, 0x3, 0x2, 0x2, 0x2, 0x5, 0x1ff, 0x3, 0x2, 0x2, + 0x2, 0x7, 0x205, 0x3, 0x2, 0x2, 0x2, 0x9, 0x20b, 0x3, 0x2, 0x2, 0x2, + 0xb, 0x20f, 0x3, 0x2, 0x2, 0x2, 0xd, 0x215, 0x3, 0x2, 0x2, 0x2, 0xf, + 0x219, 0x3, 0x2, 0x2, 0x2, 0x11, 0x21e, 0x3, 0x2, 0x2, 0x2, 0x13, 0x222, + 0x3, 0x2, 0x2, 0x2, 0x15, 0x228, 0x3, 0x2, 0x2, 0x2, 0x17, 0x239, 0x3, + 0x2, 0x2, 0x2, 0x19, 0x23b, 0x3, 0x2, 0x2, 0x2, 0x1b, 0x240, 0x3, 0x2, + 0x2, 0x2, 0x1d, 0x244, 0x3, 0x2, 0x2, 0x2, 0x1f, 0x24a, 0x3, 0x2, 0x2, + 0x2, 0x21, 0x251, 0x3, 0x2, 0x2, 0x2, 0x23, 0x259, 0x3, 0x2, 0x2, 0x2, + 0x25, 0x25e, 0x3, 0x2, 0x2, 0x2, 0x27, 0x261, 0x3, 0x2, 0x2, 0x2, 0x29, + 0x266, 0x3, 0x2, 0x2, 0x2, 0x2b, 0x26b, 0x3, 0x2, 0x2, 0x2, 0x2d, 0x271, + 0x3, 0x2, 0x2, 0x2, 0x2f, 0x277, 0x3, 0x2, 0x2, 0x2, 0x31, 0x27f, 0x3, + 0x2, 0x2, 0x2, 0x33, 0x285, 0x3, 0x2, 0x2, 0x2, 0x35, 0x28d, 0x3, 0x2, + 0x2, 0x2, 0x37, 0x294, 0x3, 0x2, 0x2, 0x2, 0x39, 0x29c, 0x3, 0x2, 0x2, + 0x2, 0x3b, 0x2a7, 0x3, 0x2, 0x2, 0x2, 0x3d, 0x2ae, 0x3, 0x2, 0x2, 0x2, + 0x3f, 0x2b4, 0x3, 0x2, 0x2, 0x2, 0x41, 0x2b9, 0x3, 0x2, 0x2, 0x2, 0x43, + 0x2c2, 0x3, 0x2, 0x2, 0x2, 0x45, 0x2cc, 0x3, 0x2, 0x2, 0x2, 0x47, 0x2d1, + 0x3, 0x2, 0x2, 0x2, 0x49, 0x2d5, 0x3, 0x2, 0x2, 0x2, 0x4b, 0x2e1, 0x3, + 0x2, 0x2, 0x2, 0x4d, 0x2e9, 0x3, 0x2, 0x2, 0x2, 0x4f, 0x2ef, 0x3, 0x2, + 0x2, 0x2, 0x51, 0x2f6, 0x3, 0x2, 0x2, 0x2, 0x53, 0x2fb, 0x3, 0x2, 0x2, + 0x2, 0x55, 0x306, 0x3, 0x2, 0x2, 0x2, 0x57, 0x30f, 0x3, 0x2, 0x2, 0x2, + 0x59, 0x316, 0x3, 0x2, 0x2, 0x2, 0x5b, 0x323, 0x3, 0x2, 0x2, 0x2, 0x5d, + 0x32e, 0x3, 0x2, 0x2, 0x2, 0x5f, 0x333, 0x3, 0x2, 0x2, 0x2, 0x61, 0x33c, + 0x3, 0x2, 0x2, 0x2, 0x63, 0x348, 0x3, 0x2, 0x2, 0x2, 0x65, 0x34d, 0x3, + 0x2, 0x2, 0x2, 0x67, 0x352, 0x3, 0x2, 0x2, 0x2, 0x69, 0x356, 0x3, 0x2, + 0x2, 0x2, 0x6b, 0x35d, 0x3, 0x2, 0x2, 0x2, 0x6d, 0x364, 0x3, 0x2, 0x2, + 0x2, 0x6f, 0x36b, 0x3, 0x2, 0x2, 0x2, 0x71, 0x373, 0x3, 0x2, 0x2, 0x2, + 0x73, 0x37e, 0x3, 0x2, 0x2, 0x2, 0x75, 0x386, 0x3, 0x2, 0x2, 0x2, 0x77, + 0x38e, 0x3, 0x2, 0x2, 0x2, 0x79, 0x394, 0x3, 0x2, 0x2, 0x2, 0x7b, 0x39a, + 0x3, 0x2, 0x2, 0x2, 0x7d, 0x3a0, 0x3, 0x2, 0x2, 0x2, 0x7f, 0x3a4, 0x3, + 0x2, 0x2, 0x2, 0x81, 0x3ab, 0x3, 0x2, 0x2, 0x2, 0x83, 0x3b2, 0x3, 0x2, + 0x2, 0x2, 0x85, 0x3b7, 0x3, 0x2, 0x2, 0x2, 0x87, 0x3bc, 0x3, 0x2, 0x2, + 0x2, 0x89, 0x3c5, 0x3, 0x2, 0x2, 0x2, 0x8b, 0x3cc, 0x3, 0x2, 0x2, 0x2, + 0x8d, 0x3d8, 0x3, 0x2, 0x2, 0x2, 0x8f, 0x3de, 0x3, 0x2, 0x2, 0x2, 0x91, + 0x3e5, 0x3, 0x2, 0x2, 0x2, 0x93, 0x3f2, 0x3, 0x2, 0x2, 0x2, 0x95, 0x3f7, + 0x3, 0x2, 0x2, 0x2, 0x97, 0x3fa, 0x3, 0x2, 0x2, 0x2, 0x99, 0x3fd, 0x3, + 0x2, 0x2, 0x2, 0x9b, 0x403, 0x3, 0x2, 0x2, 0x2, 0x9d, 0x406, 0x3, 0x2, + 0x2, 0x2, 0x9f, 0x419, 0x3, 0x2, 0x2, 0x2, 0xa1, 0x41b, 0x3, 0x2, 0x2, + 0x2, 0xa3, 0x425, 0x3, 0x2, 0x2, 0x2, 0xa5, 0x42b, 0x3, 0x2, 0x2, 0x2, + 0xa7, 0x432, 0x3, 0x2, 0x2, 0x2, 0xa9, 0x43b, 0x3, 0x2, 0x2, 0x2, 0xab, + 0x440, 0x3, 0x2, 0x2, 0x2, 0xad, 0x443, 0x3, 0x2, 0x2, 0x2, 0xaf, 0x450, + 0x3, 0x2, 0x2, 0x2, 0xb1, 0x455, 0x3, 0x2, 0x2, 0x2, 0xb3, 0x459, 0x3, + 0x2, 0x2, 0x2, 0xb5, 0x45e, 0x3, 0x2, 0x2, 0x2, 0xb7, 0x463, 0x3, 0x2, + 0x2, 0x2, 0xb9, 0x46a, 0x3, 0x2, 0x2, 0x2, 0xbb, 0x472, 0x3, 0x2, 0x2, + 0x2, 0xbd, 0x477, 0x3, 0x2, 0x2, 0x2, 0xbf, 0x480, 0x3, 0x2, 0x2, 0x2, + 0xc1, 0x485, 0x3, 0x2, 0x2, 0x2, 0xc3, 0x48b, 0x3, 0x2, 0x2, 0x2, 0xc5, + 0x490, 0x3, 0x2, 0x2, 0x2, 0xc7, 0x496, 0x3, 0x2, 0x2, 0x2, 0xc9, 0x49b, + 0x3, 0x2, 0x2, 0x2, 0xcb, 0x4a7, 0x3, 0x2, 0x2, 0x2, 0xcd, 0x4b4, 0x3, + 0x2, 0x2, 0x2, 0xcf, 0x4b8, 0x3, 0x2, 0x2, 0x2, 0xd1, 0x4bf, 0x3, 0x2, + 0x2, 0x2, 0xd3, 0x4c3, 0x3, 0x2, 0x2, 0x2, 0xd5, 0x4ca, 0x3, 0x2, 0x2, + 0x2, 0xd7, 0x4d1, 0x3, 0x2, 0x2, 0x2, 0xd9, 0x4d7, 0x3, 0x2, 0x2, 0x2, + 0xdb, 0x4dc, 0x3, 0x2, 0x2, 0x2, 0xdd, 0x4e5, 0x3, 0x2, 0x2, 0x2, 0xdf, + 0x4e9, 0x3, 0x2, 0x2, 0x2, 0xe1, 0x4ec, 0x3, 0x2, 0x2, 0x2, 0xe3, 0x4f0, + 0x3, 0x2, 0x2, 0x2, 0xe5, 0x4f5, 0x3, 0x2, 0x2, 0x2, 0xe7, 0x4fb, 0x3, + 0x2, 0x2, 0x2, 0xe9, 0x502, 0x3, 0x2, 0x2, 0x2, 0xeb, 0x505, 0x3, 0x2, + 0x2, 0x2, 0xed, 0x50e, 0x3, 0x2, 0x2, 0x2, 0xef, 0x511, 0x3, 0x2, 0x2, + 0x2, 0xf1, 0x517, 0x3, 0x2, 0x2, 0x2, 0xf3, 0x51d, 0x3, 0x2, 0x2, 0x2, + 0xf5, 0x525, 0x3, 0x2, 0x2, 0x2, 0xf7, 0x52f, 0x3, 0x2, 0x2, 0x2, 0xf9, + 0x538, 0x3, 0x2, 0x2, 0x2, 0xfb, 0x541, 0x3, 0x2, 0x2, 0x2, 0xfd, 0x549, + 0x3, 0x2, 0x2, 0x2, 0xff, 0x554, 0x3, 0x2, 0x2, 0x2, 0x101, 0x55c, 0x3, + 0x2, 0x2, 0x2, 0x103, 0x562, 0x3, 0x2, 0x2, 0x2, 0x105, 0x569, 0x3, + 0x2, 0x2, 0x2, 0x107, 0x570, 0x3, 0x2, 0x2, 0x2, 0x109, 0x577, 0x3, + 0x2, 0x2, 0x2, 0x10b, 0x57f, 0x3, 0x2, 0x2, 0x2, 0x10d, 0x587, 0x3, + 0x2, 0x2, 0x2, 0x10f, 0x592, 0x3, 0x2, 0x2, 0x2, 0x111, 0x598, 0x3, + 0x2, 0x2, 0x2, 0x113, 0x59f, 0x3, 0x2, 0x2, 0x2, 0x115, 0x5a6, 0x3, + 0x2, 0x2, 0x2, 0x117, 0x5ad, 0x3, 0x2, 0x2, 0x2, 0x119, 0x5b4, 0x3, + 0x2, 0x2, 0x2, 0x11b, 0x5b9, 0x3, 0x2, 0x2, 0x2, 0x11d, 0x5bf, 0x3, + 0x2, 0x2, 0x2, 0x11f, 0x5c3, 0x3, 0x2, 0x2, 0x2, 0x121, 0x5cc, 0x3, + 0x2, 0x2, 0x2, 0x123, 0x5d1, 0x3, 0x2, 0x2, 0x2, 0x125, 0x5d8, 0x3, + 0x2, 0x2, 0x2, 0x127, 0x5de, 0x3, 0x2, 0x2, 0x2, 0x129, 0x5e3, 0x3, + 0x2, 0x2, 0x2, 0x12b, 0x5ed, 0x3, 0x2, 0x2, 0x2, 0x12d, 0x5f2, 0x3, + 0x2, 0x2, 0x2, 0x12f, 0x5f9, 0x3, 0x2, 0x2, 0x2, 0x131, 0x600, 0x3, + 0x2, 0x2, 0x2, 0x133, 0x606, 0x3, 0x2, 0x2, 0x2, 0x135, 0x60d, 0x3, + 0x2, 0x2, 0x2, 0x137, 0x617, 0x3, 0x2, 0x2, 0x2, 0x139, 0x61c, 0x3, + 0x2, 0x2, 0x2, 0x13b, 0x621, 0x3, 0x2, 0x2, 0x2, 0x13d, 0x626, 0x3, + 0x2, 0x2, 0x2, 0x13f, 0x62e, 0x3, 0x2, 0x2, 0x2, 0x141, 0x638, 0x3, + 0x2, 0x2, 0x2, 0x143, 0x63b, 0x3, 0x2, 0x2, 0x2, 0x145, 0x63f, 0x3, + 0x2, 0x2, 0x2, 0x147, 0x646, 0x3, 0x2, 0x2, 0x2, 0x149, 0x64f, 0x3, + 0x2, 0x2, 0x2, 0x14b, 0x654, 0x3, 0x2, 0x2, 0x2, 0x14d, 0x65d, 0x3, + 0x2, 0x2, 0x2, 0x14f, 0x661, 0x3, 0x2, 0x2, 0x2, 0x151, 0x666, 0x3, + 0x2, 0x2, 0x2, 0x153, 0x66c, 0x3, 0x2, 0x2, 0x2, 0x155, 0x673, 0x3, + 0x2, 0x2, 0x2, 0x157, 0x677, 0x3, 0x2, 0x2, 0x2, 0x159, 0x67d, 0x3, + 0x2, 0x2, 0x2, 0x15b, 0x682, 0x3, 0x2, 0x2, 0x2, 0x15d, 0x689, 0x3, + 0x2, 0x2, 0x2, 0x15f, 0x68e, 0x3, 0x2, 0x2, 0x2, 0x161, 0x695, 0x3, + 0x2, 0x2, 0x2, 0x163, 0x69b, 0x3, 0x2, 0x2, 0x2, 0x165, 0x6a0, 0x3, + 0x2, 0x2, 0x2, 0x167, 0x6a5, 0x3, 0x2, 0x2, 0x2, 0x169, 0x6ab, 0x3, + 0x2, 0x2, 0x2, 0x16b, 0x6ba, 0x3, 0x2, 0x2, 0x2, 0x16d, 0x6bc, 0x3, + 0x2, 0x2, 0x2, 0x16f, 0x6c2, 0x3, 0x2, 0x2, 0x2, 0x171, 0x6f1, 0x3, + 0x2, 0x2, 0x2, 0x173, 0x73f, 0x3, 0x2, 0x2, 0x2, 0x175, 0x741, 0x3, + 0x2, 0x2, 0x2, 0x177, 0x748, 0x3, 0x2, 0x2, 0x2, 0x179, 0x74c, 0x3, + 0x2, 0x2, 0x2, 0x17b, 0x753, 0x3, 0x2, 0x2, 0x2, 0x17d, 0x762, 0x3, + 0x2, 0x2, 0x2, 0x17f, 0x764, 0x3, 0x2, 0x2, 0x2, 0x181, 0x766, 0x3, + 0x2, 0x2, 0x2, 0x183, 0x768, 0x3, 0x2, 0x2, 0x2, 0x185, 0x76a, 0x3, + 0x2, 0x2, 0x2, 0x187, 0x76c, 0x3, 0x2, 0x2, 0x2, 0x189, 0x76e, 0x3, + 0x2, 0x2, 0x2, 0x18b, 0x770, 0x3, 0x2, 0x2, 0x2, 0x18d, 0x772, 0x3, + 0x2, 0x2, 0x2, 0x18f, 0x774, 0x3, 0x2, 0x2, 0x2, 0x191, 0x776, 0x3, + 0x2, 0x2, 0x2, 0x193, 0x778, 0x3, 0x2, 0x2, 0x2, 0x195, 0x77a, 0x3, + 0x2, 0x2, 0x2, 0x197, 0x77c, 0x3, 0x2, 0x2, 0x2, 0x199, 0x77e, 0x3, + 0x2, 0x2, 0x2, 0x19b, 0x780, 0x3, 0x2, 0x2, 0x2, 0x19d, 0x782, 0x3, + 0x2, 0x2, 0x2, 0x19f, 0x784, 0x3, 0x2, 0x2, 0x2, 0x1a1, 0x786, 0x3, + 0x2, 0x2, 0x2, 0x1a3, 0x788, 0x3, 0x2, 0x2, 0x2, 0x1a5, 0x78a, 0x3, + 0x2, 0x2, 0x2, 0x1a7, 0x78c, 0x3, 0x2, 0x2, 0x2, 0x1a9, 0x78e, 0x3, + 0x2, 0x2, 0x2, 0x1ab, 0x790, 0x3, 0x2, 0x2, 0x2, 0x1ad, 0x792, 0x3, + 0x2, 0x2, 0x2, 0x1af, 0x794, 0x3, 0x2, 0x2, 0x2, 0x1b1, 0x796, 0x3, + 0x2, 0x2, 0x2, 0x1b3, 0x798, 0x3, 0x2, 0x2, 0x2, 0x1b5, 0x79a, 0x3, + 0x2, 0x2, 0x2, 0x1b7, 0x79c, 0x3, 0x2, 0x2, 0x2, 0x1b9, 0x79e, 0x3, + 0x2, 0x2, 0x2, 0x1bb, 0x7a1, 0x3, 0x2, 0x2, 0x2, 0x1bd, 0x7a3, 0x3, + 0x2, 0x2, 0x2, 0x1bf, 0x7a5, 0x3, 0x2, 0x2, 0x2, 0x1c1, 0x7a7, 0x3, + 0x2, 0x2, 0x2, 0x1c3, 0x7a9, 0x3, 0x2, 0x2, 0x2, 0x1c5, 0x7ab, 0x3, + 0x2, 0x2, 0x2, 0x1c7, 0x7ae, 0x3, 0x2, 0x2, 0x2, 0x1c9, 0x7b0, 0x3, + 0x2, 0x2, 0x2, 0x1cb, 0x7b2, 0x3, 0x2, 0x2, 0x2, 0x1cd, 0x7b5, 0x3, + 0x2, 0x2, 0x2, 0x1cf, 0x7b7, 0x3, 0x2, 0x2, 0x2, 0x1d1, 0x7ba, 0x3, + 0x2, 0x2, 0x2, 0x1d3, 0x7bc, 0x3, 0x2, 0x2, 0x2, 0x1d5, 0x7be, 0x3, + 0x2, 0x2, 0x2, 0x1d7, 0x7c0, 0x3, 0x2, 0x2, 0x2, 0x1d9, 0x7c3, 0x3, + 0x2, 0x2, 0x2, 0x1db, 0x7c5, 0x3, 0x2, 0x2, 0x2, 0x1dd, 0x7cb, 0x3, + 0x2, 0x2, 0x2, 0x1df, 0x7cd, 0x3, 0x2, 0x2, 0x2, 0x1e1, 0x7cf, 0x3, + 0x2, 0x2, 0x2, 0x1e3, 0x7d1, 0x3, 0x2, 0x2, 0x2, 0x1e5, 0x7d3, 0x3, + 0x2, 0x2, 0x2, 0x1e7, 0x7d5, 0x3, 0x2, 0x2, 0x2, 0x1e9, 0x7d7, 0x3, + 0x2, 0x2, 0x2, 0x1eb, 0x7d9, 0x3, 0x2, 0x2, 0x2, 0x1ed, 0x7db, 0x3, + 0x2, 0x2, 0x2, 0x1ef, 0x7dd, 0x3, 0x2, 0x2, 0x2, 0x1f1, 0x7df, 0x3, + 0x2, 0x2, 0x2, 0x1f3, 0x7e1, 0x3, 0x2, 0x2, 0x2, 0x1f5, 0x7e3, 0x3, + 0x2, 0x2, 0x2, 0x1f7, 0x7f1, 0x3, 0x2, 0x2, 0x2, 0x1f9, 0x7ff, 0x3, + 0x2, 0x2, 0x2, 0x1fb, 0x1fc, 0x5, 0x17d, 0xbf, 0x2, 0x1fc, 0x1fd, 0x5, + 0x183, 0xc2, 0x2, 0x1fd, 0x1fe, 0x5, 0x183, 0xc2, 0x2, 0x1fe, 0x4, 0x3, + 0x2, 0x2, 0x2, 0x1ff, 0x200, 0x5, 0x17d, 0xbf, 0x2, 0x200, 0x201, 0x5, + 0x187, 0xc4, 0x2, 0x201, 0x202, 0x5, 0x1a3, 0xd2, 0x2, 0x202, 0x203, + 0x5, 0x185, 0xc3, 0x2, 0x203, 0x204, 0x5, 0x19f, 0xd0, 0x2, 0x204, 0x6, + 0x3, 0x2, 0x2, 0x2, 0x205, 0x206, 0x5, 0x17d, 0xbf, 0x2, 0x206, 0x207, + 0x5, 0x193, 0xca, 0x2, 0x207, 0x208, 0x5, 0x18d, 0xc7, 0x2, 0x208, 0x209, + 0x5, 0x17d, 0xbf, 0x2, 0x209, 0x20a, 0x5, 0x1a1, 0xd1, 0x2, 0x20a, 0x8, + 0x3, 0x2, 0x2, 0x2, 0x20b, 0x20c, 0x5, 0x17d, 0xbf, 0x2, 0x20c, 0x20d, + 0x5, 0x193, 0xca, 0x2, 0x20d, 0x20e, 0x5, 0x193, 0xca, 0x2, 0x20e, 0xa, + 0x3, 0x2, 0x2, 0x2, 0x20f, 0x210, 0x5, 0x17d, 0xbf, 0x2, 0x210, 0x211, + 0x5, 0x193, 0xca, 0x2, 0x211, 0x212, 0x5, 0x1a3, 0xd2, 0x2, 0x212, 0x213, + 0x5, 0x185, 0xc3, 0x2, 0x213, 0x214, 0x5, 0x19f, 0xd0, 0x2, 0x214, 0xc, + 0x3, 0x2, 0x2, 0x2, 0x215, 0x216, 0x5, 0x17d, 0xbf, 0x2, 0x216, 0x217, + 0x5, 0x197, 0xcc, 0x2, 0x217, 0x218, 0x5, 0x183, 0xc2, 0x2, 0x218, 0xe, + 0x3, 0x2, 0x2, 0x2, 0x219, 0x21a, 0x5, 0x17d, 0xbf, 0x2, 0x21a, 0x21b, + 0x5, 0x197, 0xcc, 0x2, 0x21b, 0x21c, 0x5, 0x1a3, 0xd2, 0x2, 0x21c, 0x21d, + 0x5, 0x18d, 0xc7, 0x2, 0x21d, 0x10, 0x3, 0x2, 0x2, 0x2, 0x21e, 0x21f, + 0x5, 0x17d, 0xbf, 0x2, 0x21f, 0x220, 0x5, 0x197, 0xcc, 0x2, 0x220, 0x221, + 0x5, 0x1ad, 0xd7, 0x2, 0x221, 0x12, 0x3, 0x2, 0x2, 0x2, 0x222, 0x223, + 0x5, 0x17d, 0xbf, 0x2, 0x223, 0x224, 0x5, 0x19f, 0xd0, 0x2, 0x224, 0x225, + 0x5, 0x19f, 0xd0, 0x2, 0x225, 0x226, 0x5, 0x17d, 0xbf, 0x2, 0x226, 0x227, + 0x5, 0x1ad, 0xd7, 0x2, 0x227, 0x14, 0x3, 0x2, 0x2, 0x2, 0x228, 0x229, + 0x5, 0x17d, 0xbf, 0x2, 0x229, 0x22a, 0x5, 0x1a1, 0xd1, 0x2, 0x22a, 0x16, + 0x3, 0x2, 0x2, 0x2, 0x22b, 0x22c, 0x5, 0x17d, 0xbf, 0x2, 0x22c, 0x22d, + 0x5, 0x1a1, 0xd1, 0x2, 0x22d, 0x22e, 0x5, 0x181, 0xc1, 0x2, 0x22e, 0x23a, + 0x3, 0x2, 0x2, 0x2, 0x22f, 0x230, 0x5, 0x17d, 0xbf, 0x2, 0x230, 0x231, + 0x5, 0x1a1, 0xd1, 0x2, 0x231, 0x232, 0x5, 0x181, 0xc1, 0x2, 0x232, 0x233, + 0x5, 0x185, 0xc3, 0x2, 0x233, 0x234, 0x5, 0x197, 0xcc, 0x2, 0x234, 0x235, + 0x5, 0x183, 0xc2, 0x2, 0x235, 0x236, 0x5, 0x18d, 0xc7, 0x2, 0x236, 0x237, + 0x5, 0x197, 0xcc, 0x2, 0x237, 0x238, 0x5, 0x189, 0xc5, 0x2, 0x238, 0x23a, + 0x3, 0x2, 0x2, 0x2, 0x239, 0x22b, 0x3, 0x2, 0x2, 0x2, 0x239, 0x22f, + 0x3, 0x2, 0x2, 0x2, 0x23a, 0x18, 0x3, 0x2, 0x2, 0x2, 0x23b, 0x23c, 0x5, + 0x17d, 0xbf, 0x2, 0x23c, 0x23d, 0x5, 0x1a1, 0xd1, 0x2, 0x23d, 0x23e, + 0x5, 0x199, 0xcd, 0x2, 0x23e, 0x23f, 0x5, 0x187, 0xc4, 0x2, 0x23f, 0x1a, + 0x3, 0x2, 0x2, 0x2, 0x240, 0x241, 0x5, 0x17d, 0xbf, 0x2, 0x241, 0x242, + 0x5, 0x1a1, 0xd1, 0x2, 0x242, 0x243, 0x5, 0x1a3, 0xd2, 0x2, 0x243, 0x1c, + 0x3, 0x2, 0x2, 0x2, 0x244, 0x245, 0x5, 0x17d, 0xbf, 0x2, 0x245, 0x246, + 0x5, 0x1a1, 0xd1, 0x2, 0x246, 0x247, 0x5, 0x1ad, 0xd7, 0x2, 0x247, 0x248, + 0x5, 0x197, 0xcc, 0x2, 0x248, 0x249, 0x5, 0x181, 0xc1, 0x2, 0x249, 0x1e, + 0x3, 0x2, 0x2, 0x2, 0x24a, 0x24b, 0x5, 0x17d, 0xbf, 0x2, 0x24b, 0x24c, + 0x5, 0x1a3, 0xd2, 0x2, 0x24c, 0x24d, 0x5, 0x1a3, 0xd2, 0x2, 0x24d, 0x24e, + 0x5, 0x17d, 0xbf, 0x2, 0x24e, 0x24f, 0x5, 0x181, 0xc1, 0x2, 0x24f, 0x250, + 0x5, 0x18b, 0xc6, 0x2, 0x250, 0x20, 0x3, 0x2, 0x2, 0x2, 0x251, 0x252, + 0x5, 0x17f, 0xc0, 0x2, 0x252, 0x253, 0x5, 0x185, 0xc3, 0x2, 0x253, 0x254, + 0x5, 0x1a3, 0xd2, 0x2, 0x254, 0x255, 0x5, 0x1a9, 0xd5, 0x2, 0x255, 0x256, + 0x5, 0x185, 0xc3, 0x2, 0x256, 0x257, 0x5, 0x185, 0xc3, 0x2, 0x257, 0x258, + 0x5, 0x197, 0xcc, 0x2, 0x258, 0x22, 0x3, 0x2, 0x2, 0x2, 0x259, 0x25a, + 0x5, 0x17f, 0xc0, 0x2, 0x25a, 0x25b, 0x5, 0x199, 0xcd, 0x2, 0x25b, 0x25c, + 0x5, 0x1a3, 0xd2, 0x2, 0x25c, 0x25d, 0x5, 0x18b, 0xc6, 0x2, 0x25d, 0x24, + 0x3, 0x2, 0x2, 0x2, 0x25e, 0x25f, 0x5, 0x17f, 0xc0, 0x2, 0x25f, 0x260, + 0x5, 0x1ad, 0xd7, 0x2, 0x260, 0x26, 0x3, 0x2, 0x2, 0x2, 0x261, 0x262, + 0x5, 0x181, 0xc1, 0x2, 0x262, 0x263, 0x5, 0x17d, 0xbf, 0x2, 0x263, 0x264, + 0x5, 0x1a1, 0xd1, 0x2, 0x264, 0x265, 0x5, 0x185, 0xc3, 0x2, 0x265, 0x28, + 0x3, 0x2, 0x2, 0x2, 0x266, 0x267, 0x5, 0x181, 0xc1, 0x2, 0x267, 0x268, + 0x5, 0x17d, 0xbf, 0x2, 0x268, 0x269, 0x5, 0x1a1, 0xd1, 0x2, 0x269, 0x26a, + 0x5, 0x1a3, 0xd2, 0x2, 0x26a, 0x2a, 0x3, 0x2, 0x2, 0x2, 0x26b, 0x26c, + 0x5, 0x181, 0xc1, 0x2, 0x26c, 0x26d, 0x5, 0x18b, 0xc6, 0x2, 0x26d, 0x26e, + 0x5, 0x185, 0xc3, 0x2, 0x26e, 0x26f, 0x5, 0x181, 0xc1, 0x2, 0x26f, 0x270, + 0x5, 0x191, 0xc9, 0x2, 0x270, 0x2c, 0x3, 0x2, 0x2, 0x2, 0x271, 0x272, + 0x5, 0x181, 0xc1, 0x2, 0x272, 0x273, 0x5, 0x193, 0xca, 0x2, 0x273, 0x274, + 0x5, 0x185, 0xc3, 0x2, 0x274, 0x275, 0x5, 0x17d, 0xbf, 0x2, 0x275, 0x276, + 0x5, 0x19f, 0xd0, 0x2, 0x276, 0x2e, 0x3, 0x2, 0x2, 0x2, 0x277, 0x278, + 0x5, 0x181, 0xc1, 0x2, 0x278, 0x279, 0x5, 0x193, 0xca, 0x2, 0x279, 0x27a, + 0x5, 0x1a5, 0xd3, 0x2, 0x27a, 0x27b, 0x5, 0x1a1, 0xd1, 0x2, 0x27b, 0x27c, + 0x5, 0x1a3, 0xd2, 0x2, 0x27c, 0x27d, 0x5, 0x185, 0xc3, 0x2, 0x27d, 0x27e, + 0x5, 0x19f, 0xd0, 0x2, 0x27e, 0x30, 0x3, 0x2, 0x2, 0x2, 0x27f, 0x280, + 0x5, 0x181, 0xc1, 0x2, 0x280, 0x281, 0x5, 0x199, 0xcd, 0x2, 0x281, 0x282, + 0x5, 0x183, 0xc2, 0x2, 0x282, 0x283, 0x5, 0x185, 0xc3, 0x2, 0x283, 0x284, + 0x5, 0x181, 0xc1, 0x2, 0x284, 0x32, 0x3, 0x2, 0x2, 0x2, 0x285, 0x286, + 0x5, 0x181, 0xc1, 0x2, 0x286, 0x287, 0x5, 0x199, 0xcd, 0x2, 0x287, 0x288, + 0x5, 0x193, 0xca, 0x2, 0x288, 0x289, 0x5, 0x193, 0xca, 0x2, 0x289, 0x28a, + 0x5, 0x17d, 0xbf, 0x2, 0x28a, 0x28b, 0x5, 0x1a3, 0xd2, 0x2, 0x28b, 0x28c, + 0x5, 0x185, 0xc3, 0x2, 0x28c, 0x34, 0x3, 0x2, 0x2, 0x2, 0x28d, 0x28e, + 0x5, 0x181, 0xc1, 0x2, 0x28e, 0x28f, 0x5, 0x199, 0xcd, 0x2, 0x28f, 0x290, + 0x5, 0x193, 0xca, 0x2, 0x290, 0x291, 0x5, 0x1a5, 0xd3, 0x2, 0x291, 0x292, + 0x5, 0x195, 0xcb, 0x2, 0x292, 0x293, 0x5, 0x197, 0xcc, 0x2, 0x293, 0x36, + 0x3, 0x2, 0x2, 0x2, 0x294, 0x295, 0x5, 0x181, 0xc1, 0x2, 0x295, 0x296, + 0x5, 0x199, 0xcd, 0x2, 0x296, 0x297, 0x5, 0x195, 0xcb, 0x2, 0x297, 0x298, + 0x5, 0x195, 0xcb, 0x2, 0x298, 0x299, 0x5, 0x185, 0xc3, 0x2, 0x299, 0x29a, + 0x5, 0x197, 0xcc, 0x2, 0x29a, 0x29b, 0x5, 0x1a3, 0xd2, 0x2, 0x29b, 0x38, + 0x3, 0x2, 0x2, 0x2, 0x29c, 0x29d, 0x5, 0x181, 0xc1, 0x2, 0x29d, 0x29e, + 0x5, 0x199, 0xcd, 0x2, 0x29e, 0x29f, 0x5, 0x197, 0xcc, 0x2, 0x29f, 0x2a0, + 0x5, 0x1a1, 0xd1, 0x2, 0x2a0, 0x2a1, 0x5, 0x1a3, 0xd2, 0x2, 0x2a1, 0x2a2, + 0x5, 0x19f, 0xd0, 0x2, 0x2a2, 0x2a3, 0x5, 0x17d, 0xbf, 0x2, 0x2a3, 0x2a4, + 0x5, 0x18d, 0xc7, 0x2, 0x2a4, 0x2a5, 0x5, 0x197, 0xcc, 0x2, 0x2a5, 0x2a6, + 0x5, 0x1a3, 0xd2, 0x2, 0x2a6, 0x3a, 0x3, 0x2, 0x2, 0x2, 0x2a7, 0x2a8, + 0x5, 0x181, 0xc1, 0x2, 0x2a8, 0x2a9, 0x5, 0x19f, 0xd0, 0x2, 0x2a9, 0x2aa, + 0x5, 0x185, 0xc3, 0x2, 0x2aa, 0x2ab, 0x5, 0x17d, 0xbf, 0x2, 0x2ab, 0x2ac, + 0x5, 0x1a3, 0xd2, 0x2, 0x2ac, 0x2ad, 0x5, 0x185, 0xc3, 0x2, 0x2ad, 0x3c, + 0x3, 0x2, 0x2, 0x2, 0x2ae, 0x2af, 0x5, 0x181, 0xc1, 0x2, 0x2af, 0x2b0, + 0x5, 0x19f, 0xd0, 0x2, 0x2b0, 0x2b1, 0x5, 0x199, 0xcd, 0x2, 0x2b1, 0x2b2, + 0x5, 0x1a1, 0xd1, 0x2, 0x2b2, 0x2b3, 0x5, 0x1a1, 0xd1, 0x2, 0x2b3, 0x3e, + 0x3, 0x2, 0x2, 0x2, 0x2b4, 0x2b5, 0x5, 0x181, 0xc1, 0x2, 0x2b5, 0x2b6, + 0x5, 0x1a5, 0xd3, 0x2, 0x2b6, 0x2b7, 0x5, 0x17f, 0xc0, 0x2, 0x2b7, 0x2b8, + 0x5, 0x185, 0xc3, 0x2, 0x2b8, 0x40, 0x3, 0x2, 0x2, 0x2, 0x2b9, 0x2ba, + 0x5, 0x183, 0xc2, 0x2, 0x2ba, 0x2bb, 0x5, 0x17d, 0xbf, 0x2, 0x2bb, 0x2bc, + 0x5, 0x1a3, 0xd2, 0x2, 0x2bc, 0x2bd, 0x5, 0x17d, 0xbf, 0x2, 0x2bd, 0x2be, + 0x5, 0x17f, 0xc0, 0x2, 0x2be, 0x2bf, 0x5, 0x17d, 0xbf, 0x2, 0x2bf, 0x2c0, + 0x5, 0x1a1, 0xd1, 0x2, 0x2c0, 0x2c1, 0x5, 0x185, 0xc3, 0x2, 0x2c1, 0x42, + 0x3, 0x2, 0x2, 0x2, 0x2c2, 0x2c3, 0x5, 0x183, 0xc2, 0x2, 0x2c3, 0x2c4, + 0x5, 0x17d, 0xbf, 0x2, 0x2c4, 0x2c5, 0x5, 0x1a3, 0xd2, 0x2, 0x2c5, 0x2c6, + 0x5, 0x17d, 0xbf, 0x2, 0x2c6, 0x2c7, 0x5, 0x17f, 0xc0, 0x2, 0x2c7, 0x2c8, + 0x5, 0x17d, 0xbf, 0x2, 0x2c8, 0x2c9, 0x5, 0x1a1, 0xd1, 0x2, 0x2c9, 0x2ca, + 0x5, 0x185, 0xc3, 0x2, 0x2ca, 0x2cb, 0x5, 0x1a1, 0xd1, 0x2, 0x2cb, 0x44, + 0x3, 0x2, 0x2, 0x2, 0x2cc, 0x2cd, 0x5, 0x183, 0xc2, 0x2, 0x2cd, 0x2ce, + 0x5, 0x17d, 0xbf, 0x2, 0x2ce, 0x2cf, 0x5, 0x1a3, 0xd2, 0x2, 0x2cf, 0x2d0, + 0x5, 0x185, 0xc3, 0x2, 0x2d0, 0x46, 0x3, 0x2, 0x2, 0x2, 0x2d1, 0x2d2, + 0x5, 0x183, 0xc2, 0x2, 0x2d2, 0x2d3, 0x5, 0x17d, 0xbf, 0x2, 0x2d3, 0x2d4, + 0x5, 0x1ad, 0xd7, 0x2, 0x2d4, 0x48, 0x3, 0x2, 0x2, 0x2, 0x2d5, 0x2d6, + 0x5, 0x183, 0xc2, 0x2, 0x2d6, 0x2d7, 0x5, 0x185, 0xc3, 0x2, 0x2d7, 0x2d8, + 0x5, 0x183, 0xc2, 0x2, 0x2d8, 0x2d9, 0x5, 0x1a5, 0xd3, 0x2, 0x2d9, 0x2da, + 0x5, 0x19b, 0xce, 0x2, 0x2da, 0x2db, 0x5, 0x193, 0xca, 0x2, 0x2db, 0x2dc, + 0x5, 0x18d, 0xc7, 0x2, 0x2dc, 0x2dd, 0x5, 0x181, 0xc1, 0x2, 0x2dd, 0x2de, + 0x5, 0x17d, 0xbf, 0x2, 0x2de, 0x2df, 0x5, 0x1a3, 0xd2, 0x2, 0x2df, 0x2e0, + 0x5, 0x185, 0xc3, 0x2, 0x2e0, 0x4a, 0x3, 0x2, 0x2, 0x2, 0x2e1, 0x2e2, + 0x5, 0x183, 0xc2, 0x2, 0x2e2, 0x2e3, 0x5, 0x185, 0xc3, 0x2, 0x2e3, 0x2e4, + 0x5, 0x187, 0xc4, 0x2, 0x2e4, 0x2e5, 0x5, 0x17d, 0xbf, 0x2, 0x2e5, 0x2e6, + 0x5, 0x1a5, 0xd3, 0x2, 0x2e6, 0x2e7, 0x5, 0x193, 0xca, 0x2, 0x2e7, 0x2e8, + 0x5, 0x1a3, 0xd2, 0x2, 0x2e8, 0x4c, 0x3, 0x2, 0x2, 0x2, 0x2e9, 0x2ea, + 0x5, 0x183, 0xc2, 0x2, 0x2ea, 0x2eb, 0x5, 0x185, 0xc3, 0x2, 0x2eb, 0x2ec, + 0x5, 0x193, 0xca, 0x2, 0x2ec, 0x2ed, 0x5, 0x17d, 0xbf, 0x2, 0x2ed, 0x2ee, + 0x5, 0x1ad, 0xd7, 0x2, 0x2ee, 0x4e, 0x3, 0x2, 0x2, 0x2, 0x2ef, 0x2f0, + 0x5, 0x183, 0xc2, 0x2, 0x2f0, 0x2f1, 0x5, 0x185, 0xc3, 0x2, 0x2f1, 0x2f2, + 0x5, 0x193, 0xca, 0x2, 0x2f2, 0x2f3, 0x5, 0x185, 0xc3, 0x2, 0x2f3, 0x2f4, + 0x5, 0x1a3, 0xd2, 0x2, 0x2f4, 0x2f5, 0x5, 0x185, 0xc3, 0x2, 0x2f5, 0x50, + 0x3, 0x2, 0x2, 0x2, 0x2f6, 0x2f7, 0x5, 0x183, 0xc2, 0x2, 0x2f7, 0x2f8, + 0x5, 0x185, 0xc3, 0x2, 0x2f8, 0x2f9, 0x5, 0x1a1, 0xd1, 0x2, 0x2f9, 0x2fa, + 0x5, 0x181, 0xc1, 0x2, 0x2fa, 0x52, 0x3, 0x2, 0x2, 0x2, 0x2fb, 0x2fc, + 0x5, 0x183, 0xc2, 0x2, 0x2fc, 0x2fd, 0x5, 0x185, 0xc3, 0x2, 0x2fd, 0x2fe, + 0x5, 0x1a1, 0xd1, 0x2, 0x2fe, 0x2ff, 0x5, 0x181, 0xc1, 0x2, 0x2ff, 0x300, + 0x5, 0x185, 0xc3, 0x2, 0x300, 0x301, 0x5, 0x197, 0xcc, 0x2, 0x301, 0x302, + 0x5, 0x183, 0xc2, 0x2, 0x302, 0x303, 0x5, 0x18d, 0xc7, 0x2, 0x303, 0x304, + 0x5, 0x197, 0xcc, 0x2, 0x304, 0x305, 0x5, 0x189, 0xc5, 0x2, 0x305, 0x54, + 0x3, 0x2, 0x2, 0x2, 0x306, 0x307, 0x5, 0x183, 0xc2, 0x2, 0x307, 0x308, + 0x5, 0x185, 0xc3, 0x2, 0x308, 0x309, 0x5, 0x1a1, 0xd1, 0x2, 0x309, 0x30a, + 0x5, 0x181, 0xc1, 0x2, 0x30a, 0x30b, 0x5, 0x19f, 0xd0, 0x2, 0x30b, 0x30c, + 0x5, 0x18d, 0xc7, 0x2, 0x30c, 0x30d, 0x5, 0x17f, 0xc0, 0x2, 0x30d, 0x30e, + 0x5, 0x185, 0xc3, 0x2, 0x30e, 0x56, 0x3, 0x2, 0x2, 0x2, 0x30f, 0x310, + 0x5, 0x183, 0xc2, 0x2, 0x310, 0x311, 0x5, 0x185, 0xc3, 0x2, 0x311, 0x312, + 0x5, 0x1a3, 0xd2, 0x2, 0x312, 0x313, 0x5, 0x17d, 0xbf, 0x2, 0x313, 0x314, + 0x5, 0x181, 0xc1, 0x2, 0x314, 0x315, 0x5, 0x18b, 0xc6, 0x2, 0x315, 0x58, + 0x3, 0x2, 0x2, 0x2, 0x316, 0x317, 0x5, 0x183, 0xc2, 0x2, 0x317, 0x318, + 0x5, 0x18d, 0xc7, 0x2, 0x318, 0x319, 0x5, 0x181, 0xc1, 0x2, 0x319, 0x31a, + 0x5, 0x1a3, 0xd2, 0x2, 0x31a, 0x31b, 0x5, 0x18d, 0xc7, 0x2, 0x31b, 0x31c, + 0x5, 0x199, 0xcd, 0x2, 0x31c, 0x31d, 0x5, 0x197, 0xcc, 0x2, 0x31d, 0x31e, + 0x5, 0x17d, 0xbf, 0x2, 0x31e, 0x31f, 0x5, 0x19f, 0xd0, 0x2, 0x31f, 0x320, + 0x5, 0x18d, 0xc7, 0x2, 0x320, 0x321, 0x5, 0x185, 0xc3, 0x2, 0x321, 0x322, + 0x5, 0x1a1, 0xd1, 0x2, 0x322, 0x5a, 0x3, 0x2, 0x2, 0x2, 0x323, 0x324, + 0x5, 0x183, 0xc2, 0x2, 0x324, 0x325, 0x5, 0x18d, 0xc7, 0x2, 0x325, 0x326, + 0x5, 0x181, 0xc1, 0x2, 0x326, 0x327, 0x5, 0x1a3, 0xd2, 0x2, 0x327, 0x328, + 0x5, 0x18d, 0xc7, 0x2, 0x328, 0x329, 0x5, 0x199, 0xcd, 0x2, 0x329, 0x32a, + 0x5, 0x197, 0xcc, 0x2, 0x32a, 0x32b, 0x5, 0x17d, 0xbf, 0x2, 0x32b, 0x32c, + 0x5, 0x19f, 0xd0, 0x2, 0x32c, 0x32d, 0x5, 0x1ad, 0xd7, 0x2, 0x32d, 0x5c, + 0x3, 0x2, 0x2, 0x2, 0x32e, 0x32f, 0x5, 0x183, 0xc2, 0x2, 0x32f, 0x330, + 0x5, 0x18d, 0xc7, 0x2, 0x330, 0x331, 0x5, 0x1a1, 0xd1, 0x2, 0x331, 0x332, + 0x5, 0x191, 0xc9, 0x2, 0x332, 0x5e, 0x3, 0x2, 0x2, 0x2, 0x333, 0x334, + 0x5, 0x183, 0xc2, 0x2, 0x334, 0x335, 0x5, 0x18d, 0xc7, 0x2, 0x335, 0x336, + 0x5, 0x1a1, 0xd1, 0x2, 0x336, 0x337, 0x5, 0x1a3, 0xd2, 0x2, 0x337, 0x338, + 0x5, 0x18d, 0xc7, 0x2, 0x338, 0x339, 0x5, 0x197, 0xcc, 0x2, 0x339, 0x33a, + 0x5, 0x181, 0xc1, 0x2, 0x33a, 0x33b, 0x5, 0x1a3, 0xd2, 0x2, 0x33b, 0x60, + 0x3, 0x2, 0x2, 0x2, 0x33c, 0x33d, 0x5, 0x183, 0xc2, 0x2, 0x33d, 0x33e, + 0x5, 0x18d, 0xc7, 0x2, 0x33e, 0x33f, 0x5, 0x1a1, 0xd1, 0x2, 0x33f, 0x340, + 0x5, 0x1a3, 0xd2, 0x2, 0x340, 0x341, 0x5, 0x19f, 0xd0, 0x2, 0x341, 0x342, + 0x5, 0x18d, 0xc7, 0x2, 0x342, 0x343, 0x5, 0x17f, 0xc0, 0x2, 0x343, 0x344, + 0x5, 0x1a5, 0xd3, 0x2, 0x344, 0x345, 0x5, 0x1a3, 0xd2, 0x2, 0x345, 0x346, + 0x5, 0x185, 0xc3, 0x2, 0x346, 0x347, 0x5, 0x183, 0xc2, 0x2, 0x347, 0x62, + 0x3, 0x2, 0x2, 0x2, 0x348, 0x349, 0x5, 0x183, 0xc2, 0x2, 0x349, 0x34a, + 0x5, 0x19f, 0xd0, 0x2, 0x34a, 0x34b, 0x5, 0x199, 0xcd, 0x2, 0x34b, 0x34c, + 0x5, 0x19b, 0xce, 0x2, 0x34c, 0x64, 0x3, 0x2, 0x2, 0x2, 0x34d, 0x34e, + 0x5, 0x185, 0xc3, 0x2, 0x34e, 0x34f, 0x5, 0x193, 0xca, 0x2, 0x34f, 0x350, + 0x5, 0x1a1, 0xd1, 0x2, 0x350, 0x351, 0x5, 0x185, 0xc3, 0x2, 0x351, 0x66, + 0x3, 0x2, 0x2, 0x2, 0x352, 0x353, 0x5, 0x185, 0xc3, 0x2, 0x353, 0x354, + 0x5, 0x197, 0xcc, 0x2, 0x354, 0x355, 0x5, 0x183, 0xc2, 0x2, 0x355, 0x68, + 0x3, 0x2, 0x2, 0x2, 0x356, 0x357, 0x5, 0x185, 0xc3, 0x2, 0x357, 0x358, + 0x5, 0x197, 0xcc, 0x2, 0x358, 0x359, 0x5, 0x189, 0xc5, 0x2, 0x359, 0x35a, + 0x5, 0x18d, 0xc7, 0x2, 0x35a, 0x35b, 0x5, 0x197, 0xcc, 0x2, 0x35b, 0x35c, + 0x5, 0x185, 0xc3, 0x2, 0x35c, 0x6a, 0x3, 0x2, 0x2, 0x2, 0x35d, 0x35e, + 0x5, 0x185, 0xc3, 0x2, 0x35e, 0x35f, 0x5, 0x1a7, 0xd4, 0x2, 0x35f, 0x360, + 0x5, 0x185, 0xc3, 0x2, 0x360, 0x361, 0x5, 0x197, 0xcc, 0x2, 0x361, 0x362, + 0x5, 0x1a3, 0xd2, 0x2, 0x362, 0x363, 0x5, 0x1a1, 0xd1, 0x2, 0x363, 0x6c, + 0x3, 0x2, 0x2, 0x2, 0x364, 0x365, 0x5, 0x185, 0xc3, 0x2, 0x365, 0x366, + 0x5, 0x1ab, 0xd6, 0x2, 0x366, 0x367, 0x5, 0x18d, 0xc7, 0x2, 0x367, 0x368, + 0x5, 0x1a1, 0xd1, 0x2, 0x368, 0x369, 0x5, 0x1a3, 0xd2, 0x2, 0x369, 0x36a, + 0x5, 0x1a1, 0xd1, 0x2, 0x36a, 0x6e, 0x3, 0x2, 0x2, 0x2, 0x36b, 0x36c, + 0x5, 0x185, 0xc3, 0x2, 0x36c, 0x36d, 0x5, 0x1ab, 0xd6, 0x2, 0x36d, 0x36e, + 0x5, 0x19b, 0xce, 0x2, 0x36e, 0x36f, 0x5, 0x193, 0xca, 0x2, 0x36f, 0x370, + 0x5, 0x17d, 0xbf, 0x2, 0x370, 0x371, 0x5, 0x18d, 0xc7, 0x2, 0x371, 0x372, + 0x5, 0x197, 0xcc, 0x2, 0x372, 0x70, 0x3, 0x2, 0x2, 0x2, 0x373, 0x374, + 0x5, 0x185, 0xc3, 0x2, 0x374, 0x375, 0x5, 0x1ab, 0xd6, 0x2, 0x375, 0x376, + 0x5, 0x19b, 0xce, 0x2, 0x376, 0x377, 0x5, 0x19f, 0xd0, 0x2, 0x377, 0x378, + 0x5, 0x185, 0xc3, 0x2, 0x378, 0x379, 0x5, 0x1a1, 0xd1, 0x2, 0x379, 0x37a, + 0x5, 0x1a1, 0xd1, 0x2, 0x37a, 0x37b, 0x5, 0x18d, 0xc7, 0x2, 0x37b, 0x37c, + 0x5, 0x199, 0xcd, 0x2, 0x37c, 0x37d, 0x5, 0x197, 0xcc, 0x2, 0x37d, 0x72, + 0x3, 0x2, 0x2, 0x2, 0x37e, 0x37f, 0x5, 0x185, 0xc3, 0x2, 0x37f, 0x380, + 0x5, 0x1ab, 0xd6, 0x2, 0x380, 0x381, 0x5, 0x1a3, 0xd2, 0x2, 0x381, 0x382, + 0x5, 0x19f, 0xd0, 0x2, 0x382, 0x383, 0x5, 0x17d, 0xbf, 0x2, 0x383, 0x384, + 0x5, 0x181, 0xc1, 0x2, 0x384, 0x385, 0x5, 0x1a3, 0xd2, 0x2, 0x385, 0x74, + 0x3, 0x2, 0x2, 0x2, 0x386, 0x387, 0x5, 0x187, 0xc4, 0x2, 0x387, 0x388, + 0x5, 0x185, 0xc3, 0x2, 0x388, 0x389, 0x5, 0x1a3, 0xd2, 0x2, 0x389, 0x38a, + 0x5, 0x181, 0xc1, 0x2, 0x38a, 0x38b, 0x5, 0x18b, 0xc6, 0x2, 0x38b, 0x38c, + 0x5, 0x185, 0xc3, 0x2, 0x38c, 0x38d, 0x5, 0x1a1, 0xd1, 0x2, 0x38d, 0x76, + 0x3, 0x2, 0x2, 0x2, 0x38e, 0x38f, 0x5, 0x187, 0xc4, 0x2, 0x38f, 0x390, + 0x5, 0x18d, 0xc7, 0x2, 0x390, 0x391, 0x5, 0x197, 0xcc, 0x2, 0x391, 0x392, + 0x5, 0x17d, 0xbf, 0x2, 0x392, 0x393, 0x5, 0x193, 0xca, 0x2, 0x393, 0x78, + 0x3, 0x2, 0x2, 0x2, 0x394, 0x395, 0x5, 0x187, 0xc4, 0x2, 0x395, 0x396, + 0x5, 0x18d, 0xc7, 0x2, 0x396, 0x397, 0x5, 0x19f, 0xd0, 0x2, 0x397, 0x398, + 0x5, 0x1a1, 0xd1, 0x2, 0x398, 0x399, 0x5, 0x1a3, 0xd2, 0x2, 0x399, 0x7a, + 0x3, 0x2, 0x2, 0x2, 0x39a, 0x39b, 0x5, 0x187, 0xc4, 0x2, 0x39b, 0x39c, + 0x5, 0x193, 0xca, 0x2, 0x39c, 0x39d, 0x5, 0x1a5, 0xd3, 0x2, 0x39d, 0x39e, + 0x5, 0x1a1, 0xd1, 0x2, 0x39e, 0x39f, 0x5, 0x18b, 0xc6, 0x2, 0x39f, 0x7c, + 0x3, 0x2, 0x2, 0x2, 0x3a0, 0x3a1, 0x5, 0x187, 0xc4, 0x2, 0x3a1, 0x3a2, + 0x5, 0x199, 0xcd, 0x2, 0x3a2, 0x3a3, 0x5, 0x19f, 0xd0, 0x2, 0x3a3, 0x7e, + 0x3, 0x2, 0x2, 0x2, 0x3a4, 0x3a5, 0x5, 0x187, 0xc4, 0x2, 0x3a5, 0x3a6, + 0x5, 0x199, 0xcd, 0x2, 0x3a6, 0x3a7, 0x5, 0x19f, 0xd0, 0x2, 0x3a7, 0x3a8, + 0x5, 0x195, 0xcb, 0x2, 0x3a8, 0x3a9, 0x5, 0x17d, 0xbf, 0x2, 0x3a9, 0x3aa, + 0x5, 0x1a3, 0xd2, 0x2, 0x3aa, 0x80, 0x3, 0x2, 0x2, 0x2, 0x3ab, 0x3ac, + 0x5, 0x187, 0xc4, 0x2, 0x3ac, 0x3ad, 0x5, 0x19f, 0xd0, 0x2, 0x3ad, 0x3ae, + 0x5, 0x185, 0xc3, 0x2, 0x3ae, 0x3af, 0x5, 0x185, 0xc3, 0x2, 0x3af, 0x3b0, + 0x5, 0x1af, 0xd8, 0x2, 0x3b0, 0x3b1, 0x5, 0x185, 0xc3, 0x2, 0x3b1, 0x82, + 0x3, 0x2, 0x2, 0x2, 0x3b2, 0x3b3, 0x5, 0x187, 0xc4, 0x2, 0x3b3, 0x3b4, + 0x5, 0x19f, 0xd0, 0x2, 0x3b4, 0x3b5, 0x5, 0x199, 0xcd, 0x2, 0x3b5, 0x3b6, + 0x5, 0x195, 0xcb, 0x2, 0x3b6, 0x84, 0x3, 0x2, 0x2, 0x2, 0x3b7, 0x3b8, + 0x5, 0x187, 0xc4, 0x2, 0x3b8, 0x3b9, 0x5, 0x1a5, 0xd3, 0x2, 0x3b9, 0x3ba, + 0x5, 0x193, 0xca, 0x2, 0x3ba, 0x3bb, 0x5, 0x193, 0xca, 0x2, 0x3bb, 0x86, + 0x3, 0x2, 0x2, 0x2, 0x3bc, 0x3bd, 0x5, 0x187, 0xc4, 0x2, 0x3bd, 0x3be, + 0x5, 0x1a5, 0xd3, 0x2, 0x3be, 0x3bf, 0x5, 0x197, 0xcc, 0x2, 0x3bf, 0x3c0, + 0x5, 0x181, 0xc1, 0x2, 0x3c0, 0x3c1, 0x5, 0x1a3, 0xd2, 0x2, 0x3c1, 0x3c2, + 0x5, 0x18d, 0xc7, 0x2, 0x3c2, 0x3c3, 0x5, 0x199, 0xcd, 0x2, 0x3c3, 0x3c4, + 0x5, 0x197, 0xcc, 0x2, 0x3c4, 0x88, 0x3, 0x2, 0x2, 0x2, 0x3c5, 0x3c6, + 0x5, 0x189, 0xc5, 0x2, 0x3c6, 0x3c7, 0x5, 0x193, 0xca, 0x2, 0x3c7, 0x3c8, + 0x5, 0x199, 0xcd, 0x2, 0x3c8, 0x3c9, 0x5, 0x17f, 0xc0, 0x2, 0x3c9, 0x3ca, + 0x5, 0x17d, 0xbf, 0x2, 0x3ca, 0x3cb, 0x5, 0x193, 0xca, 0x2, 0x3cb, 0x8a, + 0x3, 0x2, 0x2, 0x2, 0x3cc, 0x3cd, 0x5, 0x189, 0xc5, 0x2, 0x3cd, 0x3ce, + 0x5, 0x19f, 0xd0, 0x2, 0x3ce, 0x3cf, 0x5, 0x17d, 0xbf, 0x2, 0x3cf, 0x3d0, + 0x5, 0x197, 0xcc, 0x2, 0x3d0, 0x3d1, 0x5, 0x1a5, 0xd3, 0x2, 0x3d1, 0x3d2, + 0x5, 0x193, 0xca, 0x2, 0x3d2, 0x3d3, 0x5, 0x17d, 0xbf, 0x2, 0x3d3, 0x3d4, + 0x5, 0x19f, 0xd0, 0x2, 0x3d4, 0x3d5, 0x5, 0x18d, 0xc7, 0x2, 0x3d5, 0x3d6, + 0x5, 0x1a3, 0xd2, 0x2, 0x3d6, 0x3d7, 0x5, 0x1ad, 0xd7, 0x2, 0x3d7, 0x8c, 0x3, 0x2, 0x2, 0x2, 0x3d8, 0x3d9, 0x5, 0x189, 0xc5, 0x2, 0x3d9, 0x3da, - 0x5, 0x17b, 0xbe, 0x2, 0x3da, 0x3db, 0x5, 0x1a5, 0xd3, 0x2, 0x3db, 0x3dc, - 0x5, 0x18b, 0xc6, 0x2, 0x3dc, 0x3dd, 0x5, 0x195, 0xcb, 0x2, 0x3dd, 0x3de, - 0x5, 0x187, 0xc4, 0x2, 0x3de, 0x8e, 0x3, 0x2, 0x2, 0x2, 0x3df, 0x3e0, - 0x5, 0x189, 0xc5, 0x2, 0x3e0, 0x3e1, 0x5, 0x18b, 0xc6, 0x2, 0x3e1, 0x3e2, - 0x5, 0x183, 0xc2, 0x2, 0x3e2, 0x3e3, 0x5, 0x19d, 0xcf, 0x2, 0x3e3, 0x3e4, - 0x5, 0x17b, 0xbe, 0x2, 0x3e4, 0x3e5, 0x5, 0x19d, 0xcf, 0x2, 0x3e5, 0x3e6, - 0x5, 0x17f, 0xc0, 0x2, 0x3e6, 0x3e7, 0x5, 0x189, 0xc5, 0x2, 0x3e7, 0x3e8, - 0x5, 0x18b, 0xc6, 0x2, 0x3e8, 0x3e9, 0x5, 0x17f, 0xc0, 0x2, 0x3e9, 0x3ea, - 0x5, 0x17b, 0xbe, 0x2, 0x3ea, 0x3eb, 0x5, 0x191, 0xc9, 0x2, 0x3eb, 0x90, - 0x3, 0x2, 0x2, 0x2, 0x3ec, 0x3ed, 0x5, 0x189, 0xc5, 0x2, 0x3ed, 0x3ee, - 0x5, 0x197, 0xcc, 0x2, 0x3ee, 0x3ef, 0x5, 0x1a3, 0xd2, 0x2, 0x3ef, 0x3f0, - 0x5, 0x19d, 0xcf, 0x2, 0x3f0, 0x92, 0x3, 0x2, 0x2, 0x2, 0x3f1, 0x3f2, - 0x5, 0x18b, 0xc6, 0x2, 0x3f2, 0x3f3, 0x5, 0x181, 0xc1, 0x2, 0x3f3, 0x94, - 0x3, 0x2, 0x2, 0x2, 0x3f4, 0x3f5, 0x5, 0x18b, 0xc6, 0x2, 0x3f5, 0x3f6, - 0x5, 0x185, 0xc3, 0x2, 0x3f6, 0x96, 0x3, 0x2, 0x2, 0x2, 0x3f7, 0x3f8, - 0x5, 0x18b, 0xc6, 0x2, 0x3f8, 0x3f9, 0x5, 0x191, 0xc9, 0x2, 0x3f9, 0x3fa, - 0x5, 0x18b, 0xc6, 0x2, 0x3fa, 0x3fb, 0x5, 0x18f, 0xc8, 0x2, 0x3fb, 0x3fc, - 0x5, 0x183, 0xc2, 0x2, 0x3fc, 0x98, 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3fe, - 0x5, 0x18b, 0xc6, 0x2, 0x3fe, 0x3ff, 0x5, 0x195, 0xcb, 0x2, 0x3ff, 0x9a, - 0x3, 0x2, 0x2, 0x2, 0x400, 0x401, 0x5, 0x18b, 0xc6, 0x2, 0x401, 0x402, - 0x5, 0x195, 0xcb, 0x2, 0x402, 0x403, 0x5, 0x181, 0xc1, 0x2, 0x403, 0x404, - 0x5, 0x183, 0xc2, 0x2, 0x404, 0x405, 0x5, 0x1a9, 0xd5, 0x2, 0x405, 0x9c, - 0x3, 0x2, 0x2, 0x2, 0x406, 0x407, 0x5, 0x18b, 0xc6, 0x2, 0x407, 0x408, - 0x5, 0x195, 0xcb, 0x2, 0x408, 0x409, 0x5, 0x185, 0xc3, 0x2, 0x409, 0x414, - 0x3, 0x2, 0x2, 0x2, 0x40a, 0x40b, 0x5, 0x18b, 0xc6, 0x2, 0x40b, 0x40c, - 0x5, 0x195, 0xcb, 0x2, 0x40c, 0x40d, 0x5, 0x185, 0xc3, 0x2, 0x40d, 0x40e, - 0x5, 0x18b, 0xc6, 0x2, 0x40e, 0x40f, 0x5, 0x195, 0xcb, 0x2, 0x40f, 0x410, - 0x5, 0x18b, 0xc6, 0x2, 0x410, 0x411, 0x5, 0x1a1, 0xd1, 0x2, 0x411, 0x412, - 0x5, 0x1ab, 0xd6, 0x2, 0x412, 0x414, 0x3, 0x2, 0x2, 0x2, 0x413, 0x406, - 0x3, 0x2, 0x2, 0x2, 0x413, 0x40a, 0x3, 0x2, 0x2, 0x2, 0x414, 0x9e, 0x3, - 0x2, 0x2, 0x2, 0x415, 0x416, 0x5, 0x18b, 0xc6, 0x2, 0x416, 0x417, 0x5, - 0x195, 0xcb, 0x2, 0x417, 0x418, 0x5, 0x18d, 0xc7, 0x2, 0x418, 0x419, - 0x5, 0x183, 0xc2, 0x2, 0x419, 0x41a, 0x5, 0x17f, 0xc0, 0x2, 0x41a, 0x41b, - 0x5, 0x1a1, 0xd1, 0x2, 0x41b, 0x41c, 0x5, 0x18b, 0xc6, 0x2, 0x41c, 0x41d, - 0x5, 0x1a5, 0xd3, 0x2, 0x41d, 0x41e, 0x5, 0x183, 0xc2, 0x2, 0x41e, 0xa0, - 0x3, 0x2, 0x2, 0x2, 0x41f, 0x420, 0x5, 0x18b, 0xc6, 0x2, 0x420, 0x421, - 0x5, 0x195, 0xcb, 0x2, 0x421, 0x422, 0x5, 0x195, 0xcb, 0x2, 0x422, 0x423, - 0x5, 0x183, 0xc2, 0x2, 0x423, 0x424, 0x5, 0x19d, 0xcf, 0x2, 0x424, 0xa2, - 0x3, 0x2, 0x2, 0x2, 0x425, 0x426, 0x5, 0x18b, 0xc6, 0x2, 0x426, 0x427, - 0x5, 0x195, 0xcb, 0x2, 0x427, 0x428, 0x5, 0x19f, 0xd0, 0x2, 0x428, 0x429, - 0x5, 0x183, 0xc2, 0x2, 0x429, 0x42a, 0x5, 0x19d, 0xcf, 0x2, 0x42a, 0x42b, - 0x5, 0x1a1, 0xd1, 0x2, 0x42b, 0xa4, 0x3, 0x2, 0x2, 0x2, 0x42c, 0x42d, - 0x5, 0x18b, 0xc6, 0x2, 0x42d, 0x42e, 0x5, 0x195, 0xcb, 0x2, 0x42e, 0x42f, - 0x5, 0x1a1, 0xd1, 0x2, 0x42f, 0x430, 0x5, 0x183, 0xc2, 0x2, 0x430, 0x431, - 0x5, 0x19d, 0xcf, 0x2, 0x431, 0x432, 0x5, 0x1a5, 0xd3, 0x2, 0x432, 0x433, - 0x5, 0x17b, 0xbe, 0x2, 0x433, 0x434, 0x5, 0x191, 0xc9, 0x2, 0x434, 0xa6, - 0x3, 0x2, 0x2, 0x2, 0x435, 0x436, 0x5, 0x18b, 0xc6, 0x2, 0x436, 0x437, - 0x5, 0x195, 0xcb, 0x2, 0x437, 0x438, 0x5, 0x1a1, 0xd1, 0x2, 0x438, 0x439, - 0x5, 0x197, 0xcc, 0x2, 0x439, 0xa8, 0x3, 0x2, 0x2, 0x2, 0x43a, 0x43b, - 0x5, 0x18b, 0xc6, 0x2, 0x43b, 0x43c, 0x5, 0x19f, 0xd0, 0x2, 0x43c, 0xaa, - 0x3, 0x2, 0x2, 0x2, 0x43d, 0x43e, 0x5, 0x18b, 0xc6, 0x2, 0x43e, 0x43f, - 0x5, 0x19f, 0xd0, 0x2, 0x43f, 0x440, 0x5, 0x1f1, 0xf9, 0x2, 0x440, 0x441, - 0x5, 0x197, 0xcc, 0x2, 0x441, 0x442, 0x5, 0x17d, 0xbf, 0x2, 0x442, 0x443, - 0x5, 0x18d, 0xc7, 0x2, 0x443, 0x444, 0x5, 0x183, 0xc2, 0x2, 0x444, 0x445, - 0x5, 0x17f, 0xc0, 0x2, 0x445, 0x446, 0x5, 0x1a1, 0xd1, 0x2, 0x446, 0x447, - 0x5, 0x1f1, 0xf9, 0x2, 0x447, 0x448, 0x5, 0x18b, 0xc6, 0x2, 0x448, 0x449, - 0x5, 0x181, 0xc1, 0x2, 0x449, 0xac, 0x3, 0x2, 0x2, 0x2, 0x44a, 0x44b, - 0x5, 0x18d, 0xc7, 0x2, 0x44b, 0x44c, 0x5, 0x197, 0xcc, 0x2, 0x44c, 0x44d, - 0x5, 0x18b, 0xc6, 0x2, 0x44d, 0x44e, 0x5, 0x195, 0xcb, 0x2, 0x44e, 0xae, - 0x3, 0x2, 0x2, 0x2, 0x44f, 0x450, 0x5, 0x18f, 0xc8, 0x2, 0x450, 0x451, - 0x5, 0x183, 0xc2, 0x2, 0x451, 0x452, 0x5, 0x1ab, 0xd6, 0x2, 0x452, 0xb0, - 0x3, 0x2, 0x2, 0x2, 0x453, 0x454, 0x5, 0x18f, 0xc8, 0x2, 0x454, 0x455, - 0x5, 0x18b, 0xc6, 0x2, 0x455, 0x456, 0x5, 0x191, 0xc9, 0x2, 0x456, 0x457, - 0x5, 0x191, 0xc9, 0x2, 0x457, 0xb2, 0x3, 0x2, 0x2, 0x2, 0x458, 0x459, - 0x5, 0x191, 0xc9, 0x2, 0x459, 0x45a, 0x5, 0x17b, 0xbe, 0x2, 0x45a, 0x45b, - 0x5, 0x19f, 0xd0, 0x2, 0x45b, 0x45c, 0x5, 0x1a1, 0xd1, 0x2, 0x45c, 0xb4, - 0x3, 0x2, 0x2, 0x2, 0x45d, 0x45e, 0x5, 0x191, 0xc9, 0x2, 0x45e, 0x45f, - 0x5, 0x17b, 0xbe, 0x2, 0x45f, 0x460, 0x5, 0x1ab, 0xd6, 0x2, 0x460, 0x461, - 0x5, 0x197, 0xcc, 0x2, 0x461, 0x462, 0x5, 0x1a3, 0xd2, 0x2, 0x462, 0x463, - 0x5, 0x1a1, 0xd1, 0x2, 0x463, 0xb6, 0x3, 0x2, 0x2, 0x2, 0x464, 0x465, - 0x5, 0x191, 0xc9, 0x2, 0x465, 0x466, 0x5, 0x183, 0xc2, 0x2, 0x466, 0x467, - 0x5, 0x17b, 0xbe, 0x2, 0x467, 0x468, 0x5, 0x181, 0xc1, 0x2, 0x468, 0x469, - 0x5, 0x18b, 0xc6, 0x2, 0x469, 0x46a, 0x5, 0x195, 0xcb, 0x2, 0x46a, 0x46b, - 0x5, 0x187, 0xc4, 0x2, 0x46b, 0xb8, 0x3, 0x2, 0x2, 0x2, 0x46c, 0x46d, - 0x5, 0x191, 0xc9, 0x2, 0x46d, 0x46e, 0x5, 0x183, 0xc2, 0x2, 0x46e, 0x46f, - 0x5, 0x185, 0xc3, 0x2, 0x46f, 0x470, 0x5, 0x1a1, 0xd1, 0x2, 0x470, 0xba, - 0x3, 0x2, 0x2, 0x2, 0x471, 0x472, 0x5, 0x191, 0xc9, 0x2, 0x472, 0x473, - 0x5, 0x18b, 0xc6, 0x2, 0x473, 0x474, 0x5, 0x185, 0xc3, 0x2, 0x474, 0x475, - 0x5, 0x183, 0xc2, 0x2, 0x475, 0x476, 0x5, 0x1a1, 0xd1, 0x2, 0x476, 0x477, - 0x5, 0x18b, 0xc6, 0x2, 0x477, 0x478, 0x5, 0x193, 0xca, 0x2, 0x478, 0x479, - 0x5, 0x183, 0xc2, 0x2, 0x479, 0xbc, 0x3, 0x2, 0x2, 0x2, 0x47a, 0x47b, - 0x5, 0x191, 0xc9, 0x2, 0x47b, 0x47c, 0x5, 0x18b, 0xc6, 0x2, 0x47c, 0x47d, - 0x5, 0x18f, 0xc8, 0x2, 0x47d, 0x47e, 0x5, 0x183, 0xc2, 0x2, 0x47e, 0xbe, - 0x3, 0x2, 0x2, 0x2, 0x47f, 0x480, 0x5, 0x191, 0xc9, 0x2, 0x480, 0x481, - 0x5, 0x18b, 0xc6, 0x2, 0x481, 0x482, 0x5, 0x193, 0xca, 0x2, 0x482, 0x483, - 0x5, 0x18b, 0xc6, 0x2, 0x483, 0x484, 0x5, 0x1a1, 0xd1, 0x2, 0x484, 0xc0, - 0x3, 0x2, 0x2, 0x2, 0x485, 0x486, 0x5, 0x191, 0xc9, 0x2, 0x486, 0x487, - 0x5, 0x18b, 0xc6, 0x2, 0x487, 0x488, 0x5, 0x1a5, 0xd3, 0x2, 0x488, 0x489, - 0x5, 0x183, 0xc2, 0x2, 0x489, 0xc2, 0x3, 0x2, 0x2, 0x2, 0x48a, 0x48b, - 0x5, 0x191, 0xc9, 0x2, 0x48b, 0x48c, 0x5, 0x197, 0xcc, 0x2, 0x48c, 0x48d, - 0x5, 0x17f, 0xc0, 0x2, 0x48d, 0x48e, 0x5, 0x17b, 0xbe, 0x2, 0x48e, 0x48f, - 0x5, 0x191, 0xc9, 0x2, 0x48f, 0xc4, 0x3, 0x2, 0x2, 0x2, 0x490, 0x491, - 0x5, 0x191, 0xc9, 0x2, 0x491, 0x492, 0x5, 0x197, 0xcc, 0x2, 0x492, 0x493, - 0x5, 0x187, 0xc4, 0x2, 0x493, 0x494, 0x5, 0x19f, 0xd0, 0x2, 0x494, 0xc6, - 0x3, 0x2, 0x2, 0x2, 0x495, 0x496, 0x5, 0x193, 0xca, 0x2, 0x496, 0x497, - 0x5, 0x17b, 0xbe, 0x2, 0x497, 0x498, 0x5, 0x1a1, 0xd1, 0x2, 0x498, 0x499, - 0x5, 0x183, 0xc2, 0x2, 0x499, 0x49a, 0x5, 0x19d, 0xcf, 0x2, 0x49a, 0x49b, - 0x5, 0x18b, 0xc6, 0x2, 0x49b, 0x49c, 0x5, 0x17b, 0xbe, 0x2, 0x49c, 0x49d, - 0x5, 0x191, 0xc9, 0x2, 0x49d, 0x49e, 0x5, 0x18b, 0xc6, 0x2, 0x49e, 0x49f, - 0x5, 0x1ad, 0xd7, 0x2, 0x49f, 0x4a0, 0x5, 0x183, 0xc2, 0x2, 0x4a0, 0x4a1, - 0x5, 0x181, 0xc1, 0x2, 0x4a1, 0xc8, 0x3, 0x2, 0x2, 0x2, 0x4a2, 0x4a3, - 0x5, 0x193, 0xca, 0x2, 0x4a3, 0x4a4, 0x5, 0x17b, 0xbe, 0x2, 0x4a4, 0x4a5, - 0x5, 0x1a1, 0xd1, 0x2, 0x4a5, 0x4a6, 0x5, 0x183, 0xc2, 0x2, 0x4a6, 0x4a7, - 0x5, 0x19d, 0xcf, 0x2, 0x4a7, 0x4a8, 0x5, 0x18b, 0xc6, 0x2, 0x4a8, 0x4a9, - 0x5, 0x17b, 0xbe, 0x2, 0x4a9, 0x4aa, 0x5, 0x191, 0xc9, 0x2, 0x4aa, 0x4ab, - 0x5, 0x18b, 0xc6, 0x2, 0x4ab, 0x4ac, 0x5, 0x1ad, 0xd7, 0x2, 0x4ac, 0x4ad, - 0x5, 0x183, 0xc2, 0x2, 0x4ad, 0xca, 0x3, 0x2, 0x2, 0x2, 0x4ae, 0x4af, - 0x5, 0x193, 0xca, 0x2, 0x4af, 0x4b0, 0x5, 0x17b, 0xbe, 0x2, 0x4b0, 0x4b1, - 0x5, 0x1a9, 0xd5, 0x2, 0x4b1, 0xcc, 0x3, 0x2, 0x2, 0x2, 0x4b2, 0x4b3, - 0x5, 0x193, 0xca, 0x2, 0x4b3, 0x4b4, 0x5, 0x183, 0xc2, 0x2, 0x4b4, 0x4b5, - 0x5, 0x19d, 0xcf, 0x2, 0x4b5, 0x4b6, 0x5, 0x187, 0xc4, 0x2, 0x4b6, 0x4b7, - 0x5, 0x183, 0xc2, 0x2, 0x4b7, 0x4b8, 0x5, 0x19f, 0xd0, 0x2, 0x4b8, 0xce, - 0x3, 0x2, 0x2, 0x2, 0x4b9, 0x4ba, 0x5, 0x193, 0xca, 0x2, 0x4ba, 0x4bb, - 0x5, 0x18b, 0xc6, 0x2, 0x4bb, 0x4bc, 0x5, 0x195, 0xcb, 0x2, 0x4bc, 0xd0, - 0x3, 0x2, 0x2, 0x2, 0x4bd, 0x4be, 0x5, 0x193, 0xca, 0x2, 0x4be, 0x4bf, - 0x5, 0x18b, 0xc6, 0x2, 0x4bf, 0x4c0, 0x5, 0x195, 0xcb, 0x2, 0x4c0, 0x4c1, - 0x5, 0x1a3, 0xd2, 0x2, 0x4c1, 0x4c2, 0x5, 0x1a1, 0xd1, 0x2, 0x4c2, 0x4c3, - 0x5, 0x183, 0xc2, 0x2, 0x4c3, 0xd2, 0x3, 0x2, 0x2, 0x2, 0x4c4, 0x4c5, - 0x5, 0x193, 0xca, 0x2, 0x4c5, 0x4c6, 0x5, 0x197, 0xcc, 0x2, 0x4c6, 0x4c7, - 0x5, 0x181, 0xc1, 0x2, 0x4c7, 0x4c8, 0x5, 0x18b, 0xc6, 0x2, 0x4c8, 0x4c9, - 0x5, 0x185, 0xc3, 0x2, 0x4c9, 0x4ca, 0x5, 0x1ab, 0xd6, 0x2, 0x4ca, 0xd4, - 0x3, 0x2, 0x2, 0x2, 0x4cb, 0x4cc, 0x5, 0x193, 0xca, 0x2, 0x4cc, 0x4cd, - 0x5, 0x197, 0xcc, 0x2, 0x4cd, 0x4ce, 0x5, 0x195, 0xcb, 0x2, 0x4ce, 0x4cf, - 0x5, 0x1a1, 0xd1, 0x2, 0x4cf, 0x4d0, 0x5, 0x189, 0xc5, 0x2, 0x4d0, 0xd6, - 0x3, 0x2, 0x2, 0x2, 0x4d1, 0x4d2, 0x5, 0x193, 0xca, 0x2, 0x4d2, 0x4d3, - 0x5, 0x197, 0xcc, 0x2, 0x4d3, 0x4d4, 0x5, 0x1a5, 0xd3, 0x2, 0x4d4, 0x4d5, - 0x5, 0x183, 0xc2, 0x2, 0x4d5, 0xd8, 0x3, 0x2, 0x2, 0x2, 0x4d6, 0x4d7, - 0x5, 0x193, 0xca, 0x2, 0x4d7, 0x4d8, 0x5, 0x1a3, 0xd2, 0x2, 0x4d8, 0x4d9, - 0x5, 0x1a1, 0xd1, 0x2, 0x4d9, 0x4da, 0x5, 0x17b, 0xbe, 0x2, 0x4da, 0x4db, - 0x5, 0x1a1, 0xd1, 0x2, 0x4db, 0x4dc, 0x5, 0x18b, 0xc6, 0x2, 0x4dc, 0x4dd, - 0x5, 0x197, 0xcc, 0x2, 0x4dd, 0x4de, 0x5, 0x195, 0xcb, 0x2, 0x4de, 0xda, - 0x3, 0x2, 0x2, 0x2, 0x4df, 0x4e0, 0x5, 0x195, 0xcb, 0x2, 0x4e0, 0x4e1, - 0x5, 0x17b, 0xbe, 0x2, 0x4e1, 0x4e2, 0x5, 0x195, 0xcb, 0x2, 0x4e2, 0xdc, - 0x3, 0x2, 0x2, 0x2, 0x4e3, 0x4e4, 0x5, 0x195, 0xcb, 0x2, 0x4e4, 0x4e5, - 0x5, 0x197, 0xcc, 0x2, 0x4e5, 0xde, 0x3, 0x2, 0x2, 0x2, 0x4e6, 0x4e7, - 0x5, 0x195, 0xcb, 0x2, 0x4e7, 0x4e8, 0x5, 0x197, 0xcc, 0x2, 0x4e8, 0x4e9, - 0x5, 0x1a1, 0xd1, 0x2, 0x4e9, 0xe0, 0x3, 0x2, 0x2, 0x2, 0x4ea, 0x4eb, - 0x5, 0x195, 0xcb, 0x2, 0x4eb, 0x4ec, 0x5, 0x1a3, 0xd2, 0x2, 0x4ec, 0x4ed, - 0x5, 0x191, 0xc9, 0x2, 0x4ed, 0x4ee, 0x5, 0x191, 0xc9, 0x2, 0x4ee, 0xe2, - 0x3, 0x2, 0x2, 0x2, 0x4ef, 0x4f0, 0x5, 0x195, 0xcb, 0x2, 0x4f0, 0x4f1, - 0x5, 0x1a3, 0xd2, 0x2, 0x4f1, 0x4f2, 0x5, 0x191, 0xc9, 0x2, 0x4f2, 0x4f3, - 0x5, 0x191, 0xc9, 0x2, 0x4f3, 0x4f4, 0x5, 0x19f, 0xd0, 0x2, 0x4f4, 0xe4, + 0x5, 0x19f, 0xd0, 0x2, 0x3da, 0x3db, 0x5, 0x199, 0xcd, 0x2, 0x3db, 0x3dc, + 0x5, 0x1a5, 0xd3, 0x2, 0x3dc, 0x3dd, 0x5, 0x19b, 0xce, 0x2, 0x3dd, 0x8e, + 0x3, 0x2, 0x2, 0x2, 0x3de, 0x3df, 0x5, 0x18b, 0xc6, 0x2, 0x3df, 0x3e0, + 0x5, 0x17d, 0xbf, 0x2, 0x3e0, 0x3e1, 0x5, 0x1a7, 0xd4, 0x2, 0x3e1, 0x3e2, + 0x5, 0x18d, 0xc7, 0x2, 0x3e2, 0x3e3, 0x5, 0x197, 0xcc, 0x2, 0x3e3, 0x3e4, + 0x5, 0x189, 0xc5, 0x2, 0x3e4, 0x90, 0x3, 0x2, 0x2, 0x2, 0x3e5, 0x3e6, + 0x5, 0x18b, 0xc6, 0x2, 0x3e6, 0x3e7, 0x5, 0x18d, 0xc7, 0x2, 0x3e7, 0x3e8, + 0x5, 0x185, 0xc3, 0x2, 0x3e8, 0x3e9, 0x5, 0x19f, 0xd0, 0x2, 0x3e9, 0x3ea, + 0x5, 0x17d, 0xbf, 0x2, 0x3ea, 0x3eb, 0x5, 0x19f, 0xd0, 0x2, 0x3eb, 0x3ec, + 0x5, 0x181, 0xc1, 0x2, 0x3ec, 0x3ed, 0x5, 0x18b, 0xc6, 0x2, 0x3ed, 0x3ee, + 0x5, 0x18d, 0xc7, 0x2, 0x3ee, 0x3ef, 0x5, 0x181, 0xc1, 0x2, 0x3ef, 0x3f0, + 0x5, 0x17d, 0xbf, 0x2, 0x3f0, 0x3f1, 0x5, 0x193, 0xca, 0x2, 0x3f1, 0x92, + 0x3, 0x2, 0x2, 0x2, 0x3f2, 0x3f3, 0x5, 0x18b, 0xc6, 0x2, 0x3f3, 0x3f4, + 0x5, 0x199, 0xcd, 0x2, 0x3f4, 0x3f5, 0x5, 0x1a5, 0xd3, 0x2, 0x3f5, 0x3f6, + 0x5, 0x19f, 0xd0, 0x2, 0x3f6, 0x94, 0x3, 0x2, 0x2, 0x2, 0x3f7, 0x3f8, + 0x5, 0x18d, 0xc7, 0x2, 0x3f8, 0x3f9, 0x5, 0x183, 0xc2, 0x2, 0x3f9, 0x96, + 0x3, 0x2, 0x2, 0x2, 0x3fa, 0x3fb, 0x5, 0x18d, 0xc7, 0x2, 0x3fb, 0x3fc, + 0x5, 0x187, 0xc4, 0x2, 0x3fc, 0x98, 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3fe, + 0x5, 0x18d, 0xc7, 0x2, 0x3fe, 0x3ff, 0x5, 0x193, 0xca, 0x2, 0x3ff, 0x400, + 0x5, 0x18d, 0xc7, 0x2, 0x400, 0x401, 0x5, 0x191, 0xc9, 0x2, 0x401, 0x402, + 0x5, 0x185, 0xc3, 0x2, 0x402, 0x9a, 0x3, 0x2, 0x2, 0x2, 0x403, 0x404, + 0x5, 0x18d, 0xc7, 0x2, 0x404, 0x405, 0x5, 0x197, 0xcc, 0x2, 0x405, 0x9c, + 0x3, 0x2, 0x2, 0x2, 0x406, 0x407, 0x5, 0x18d, 0xc7, 0x2, 0x407, 0x408, + 0x5, 0x197, 0xcc, 0x2, 0x408, 0x409, 0x5, 0x183, 0xc2, 0x2, 0x409, 0x40a, + 0x5, 0x185, 0xc3, 0x2, 0x40a, 0x40b, 0x5, 0x1ab, 0xd6, 0x2, 0x40b, 0x9e, + 0x3, 0x2, 0x2, 0x2, 0x40c, 0x40d, 0x5, 0x18d, 0xc7, 0x2, 0x40d, 0x40e, + 0x5, 0x197, 0xcc, 0x2, 0x40e, 0x40f, 0x5, 0x187, 0xc4, 0x2, 0x40f, 0x41a, + 0x3, 0x2, 0x2, 0x2, 0x410, 0x411, 0x5, 0x18d, 0xc7, 0x2, 0x411, 0x412, + 0x5, 0x197, 0xcc, 0x2, 0x412, 0x413, 0x5, 0x187, 0xc4, 0x2, 0x413, 0x414, + 0x5, 0x18d, 0xc7, 0x2, 0x414, 0x415, 0x5, 0x197, 0xcc, 0x2, 0x415, 0x416, + 0x5, 0x18d, 0xc7, 0x2, 0x416, 0x417, 0x5, 0x1a3, 0xd2, 0x2, 0x417, 0x418, + 0x5, 0x1ad, 0xd7, 0x2, 0x418, 0x41a, 0x3, 0x2, 0x2, 0x2, 0x419, 0x40c, + 0x3, 0x2, 0x2, 0x2, 0x419, 0x410, 0x3, 0x2, 0x2, 0x2, 0x41a, 0xa0, 0x3, + 0x2, 0x2, 0x2, 0x41b, 0x41c, 0x5, 0x18d, 0xc7, 0x2, 0x41c, 0x41d, 0x5, + 0x197, 0xcc, 0x2, 0x41d, 0x41e, 0x5, 0x18f, 0xc8, 0x2, 0x41e, 0x41f, + 0x5, 0x185, 0xc3, 0x2, 0x41f, 0x420, 0x5, 0x181, 0xc1, 0x2, 0x420, 0x421, + 0x5, 0x1a3, 0xd2, 0x2, 0x421, 0x422, 0x5, 0x18d, 0xc7, 0x2, 0x422, 0x423, + 0x5, 0x1a7, 0xd4, 0x2, 0x423, 0x424, 0x5, 0x185, 0xc3, 0x2, 0x424, 0xa2, + 0x3, 0x2, 0x2, 0x2, 0x425, 0x426, 0x5, 0x18d, 0xc7, 0x2, 0x426, 0x427, + 0x5, 0x197, 0xcc, 0x2, 0x427, 0x428, 0x5, 0x197, 0xcc, 0x2, 0x428, 0x429, + 0x5, 0x185, 0xc3, 0x2, 0x429, 0x42a, 0x5, 0x19f, 0xd0, 0x2, 0x42a, 0xa4, + 0x3, 0x2, 0x2, 0x2, 0x42b, 0x42c, 0x5, 0x18d, 0xc7, 0x2, 0x42c, 0x42d, + 0x5, 0x197, 0xcc, 0x2, 0x42d, 0x42e, 0x5, 0x1a1, 0xd1, 0x2, 0x42e, 0x42f, + 0x5, 0x185, 0xc3, 0x2, 0x42f, 0x430, 0x5, 0x19f, 0xd0, 0x2, 0x430, 0x431, + 0x5, 0x1a3, 0xd2, 0x2, 0x431, 0xa6, 0x3, 0x2, 0x2, 0x2, 0x432, 0x433, + 0x5, 0x18d, 0xc7, 0x2, 0x433, 0x434, 0x5, 0x197, 0xcc, 0x2, 0x434, 0x435, + 0x5, 0x1a3, 0xd2, 0x2, 0x435, 0x436, 0x5, 0x185, 0xc3, 0x2, 0x436, 0x437, + 0x5, 0x19f, 0xd0, 0x2, 0x437, 0x438, 0x5, 0x1a7, 0xd4, 0x2, 0x438, 0x439, + 0x5, 0x17d, 0xbf, 0x2, 0x439, 0x43a, 0x5, 0x193, 0xca, 0x2, 0x43a, 0xa8, + 0x3, 0x2, 0x2, 0x2, 0x43b, 0x43c, 0x5, 0x18d, 0xc7, 0x2, 0x43c, 0x43d, + 0x5, 0x197, 0xcc, 0x2, 0x43d, 0x43e, 0x5, 0x1a3, 0xd2, 0x2, 0x43e, 0x43f, + 0x5, 0x199, 0xcd, 0x2, 0x43f, 0xaa, 0x3, 0x2, 0x2, 0x2, 0x440, 0x441, + 0x5, 0x18d, 0xc7, 0x2, 0x441, 0x442, 0x5, 0x1a1, 0xd1, 0x2, 0x442, 0xac, + 0x3, 0x2, 0x2, 0x2, 0x443, 0x444, 0x5, 0x18d, 0xc7, 0x2, 0x444, 0x445, + 0x5, 0x1a1, 0xd1, 0x2, 0x445, 0x446, 0x5, 0x1f3, 0xfa, 0x2, 0x446, 0x447, + 0x5, 0x199, 0xcd, 0x2, 0x447, 0x448, 0x5, 0x17f, 0xc0, 0x2, 0x448, 0x449, + 0x5, 0x18f, 0xc8, 0x2, 0x449, 0x44a, 0x5, 0x185, 0xc3, 0x2, 0x44a, 0x44b, + 0x5, 0x181, 0xc1, 0x2, 0x44b, 0x44c, 0x5, 0x1a3, 0xd2, 0x2, 0x44c, 0x44d, + 0x5, 0x1f3, 0xfa, 0x2, 0x44d, 0x44e, 0x5, 0x18d, 0xc7, 0x2, 0x44e, 0x44f, + 0x5, 0x183, 0xc2, 0x2, 0x44f, 0xae, 0x3, 0x2, 0x2, 0x2, 0x450, 0x451, + 0x5, 0x18f, 0xc8, 0x2, 0x451, 0x452, 0x5, 0x199, 0xcd, 0x2, 0x452, 0x453, + 0x5, 0x18d, 0xc7, 0x2, 0x453, 0x454, 0x5, 0x197, 0xcc, 0x2, 0x454, 0xb0, + 0x3, 0x2, 0x2, 0x2, 0x455, 0x456, 0x5, 0x191, 0xc9, 0x2, 0x456, 0x457, + 0x5, 0x185, 0xc3, 0x2, 0x457, 0x458, 0x5, 0x1ad, 0xd7, 0x2, 0x458, 0xb2, + 0x3, 0x2, 0x2, 0x2, 0x459, 0x45a, 0x5, 0x191, 0xc9, 0x2, 0x45a, 0x45b, + 0x5, 0x18d, 0xc7, 0x2, 0x45b, 0x45c, 0x5, 0x193, 0xca, 0x2, 0x45c, 0x45d, + 0x5, 0x193, 0xca, 0x2, 0x45d, 0xb4, 0x3, 0x2, 0x2, 0x2, 0x45e, 0x45f, + 0x5, 0x193, 0xca, 0x2, 0x45f, 0x460, 0x5, 0x17d, 0xbf, 0x2, 0x460, 0x461, + 0x5, 0x1a1, 0xd1, 0x2, 0x461, 0x462, 0x5, 0x1a3, 0xd2, 0x2, 0x462, 0xb6, + 0x3, 0x2, 0x2, 0x2, 0x463, 0x464, 0x5, 0x193, 0xca, 0x2, 0x464, 0x465, + 0x5, 0x17d, 0xbf, 0x2, 0x465, 0x466, 0x5, 0x1ad, 0xd7, 0x2, 0x466, 0x467, + 0x5, 0x199, 0xcd, 0x2, 0x467, 0x468, 0x5, 0x1a5, 0xd3, 0x2, 0x468, 0x469, + 0x5, 0x1a3, 0xd2, 0x2, 0x469, 0xb8, 0x3, 0x2, 0x2, 0x2, 0x46a, 0x46b, + 0x5, 0x193, 0xca, 0x2, 0x46b, 0x46c, 0x5, 0x185, 0xc3, 0x2, 0x46c, 0x46d, + 0x5, 0x17d, 0xbf, 0x2, 0x46d, 0x46e, 0x5, 0x183, 0xc2, 0x2, 0x46e, 0x46f, + 0x5, 0x18d, 0xc7, 0x2, 0x46f, 0x470, 0x5, 0x197, 0xcc, 0x2, 0x470, 0x471, + 0x5, 0x189, 0xc5, 0x2, 0x471, 0xba, 0x3, 0x2, 0x2, 0x2, 0x472, 0x473, + 0x5, 0x193, 0xca, 0x2, 0x473, 0x474, 0x5, 0x185, 0xc3, 0x2, 0x474, 0x475, + 0x5, 0x187, 0xc4, 0x2, 0x475, 0x476, 0x5, 0x1a3, 0xd2, 0x2, 0x476, 0xbc, + 0x3, 0x2, 0x2, 0x2, 0x477, 0x478, 0x5, 0x193, 0xca, 0x2, 0x478, 0x479, + 0x5, 0x18d, 0xc7, 0x2, 0x479, 0x47a, 0x5, 0x187, 0xc4, 0x2, 0x47a, 0x47b, + 0x5, 0x185, 0xc3, 0x2, 0x47b, 0x47c, 0x5, 0x1a3, 0xd2, 0x2, 0x47c, 0x47d, + 0x5, 0x18d, 0xc7, 0x2, 0x47d, 0x47e, 0x5, 0x195, 0xcb, 0x2, 0x47e, 0x47f, + 0x5, 0x185, 0xc3, 0x2, 0x47f, 0xbe, 0x3, 0x2, 0x2, 0x2, 0x480, 0x481, + 0x5, 0x193, 0xca, 0x2, 0x481, 0x482, 0x5, 0x18d, 0xc7, 0x2, 0x482, 0x483, + 0x5, 0x191, 0xc9, 0x2, 0x483, 0x484, 0x5, 0x185, 0xc3, 0x2, 0x484, 0xc0, + 0x3, 0x2, 0x2, 0x2, 0x485, 0x486, 0x5, 0x193, 0xca, 0x2, 0x486, 0x487, + 0x5, 0x18d, 0xc7, 0x2, 0x487, 0x488, 0x5, 0x195, 0xcb, 0x2, 0x488, 0x489, + 0x5, 0x18d, 0xc7, 0x2, 0x489, 0x48a, 0x5, 0x1a3, 0xd2, 0x2, 0x48a, 0xc2, + 0x3, 0x2, 0x2, 0x2, 0x48b, 0x48c, 0x5, 0x193, 0xca, 0x2, 0x48c, 0x48d, + 0x5, 0x18d, 0xc7, 0x2, 0x48d, 0x48e, 0x5, 0x1a7, 0xd4, 0x2, 0x48e, 0x48f, + 0x5, 0x185, 0xc3, 0x2, 0x48f, 0xc4, 0x3, 0x2, 0x2, 0x2, 0x490, 0x491, + 0x5, 0x193, 0xca, 0x2, 0x491, 0x492, 0x5, 0x199, 0xcd, 0x2, 0x492, 0x493, + 0x5, 0x181, 0xc1, 0x2, 0x493, 0x494, 0x5, 0x17d, 0xbf, 0x2, 0x494, 0x495, + 0x5, 0x193, 0xca, 0x2, 0x495, 0xc6, 0x3, 0x2, 0x2, 0x2, 0x496, 0x497, + 0x5, 0x193, 0xca, 0x2, 0x497, 0x498, 0x5, 0x199, 0xcd, 0x2, 0x498, 0x499, + 0x5, 0x189, 0xc5, 0x2, 0x499, 0x49a, 0x5, 0x1a1, 0xd1, 0x2, 0x49a, 0xc8, + 0x3, 0x2, 0x2, 0x2, 0x49b, 0x49c, 0x5, 0x195, 0xcb, 0x2, 0x49c, 0x49d, + 0x5, 0x17d, 0xbf, 0x2, 0x49d, 0x49e, 0x5, 0x1a3, 0xd2, 0x2, 0x49e, 0x49f, + 0x5, 0x185, 0xc3, 0x2, 0x49f, 0x4a0, 0x5, 0x19f, 0xd0, 0x2, 0x4a0, 0x4a1, + 0x5, 0x18d, 0xc7, 0x2, 0x4a1, 0x4a2, 0x5, 0x17d, 0xbf, 0x2, 0x4a2, 0x4a3, + 0x5, 0x193, 0xca, 0x2, 0x4a3, 0x4a4, 0x5, 0x18d, 0xc7, 0x2, 0x4a4, 0x4a5, + 0x5, 0x1af, 0xd8, 0x2, 0x4a5, 0x4a6, 0x5, 0x185, 0xc3, 0x2, 0x4a6, 0xca, + 0x3, 0x2, 0x2, 0x2, 0x4a7, 0x4a8, 0x5, 0x195, 0xcb, 0x2, 0x4a8, 0x4a9, + 0x5, 0x17d, 0xbf, 0x2, 0x4a9, 0x4aa, 0x5, 0x1a3, 0xd2, 0x2, 0x4aa, 0x4ab, + 0x5, 0x185, 0xc3, 0x2, 0x4ab, 0x4ac, 0x5, 0x19f, 0xd0, 0x2, 0x4ac, 0x4ad, + 0x5, 0x18d, 0xc7, 0x2, 0x4ad, 0x4ae, 0x5, 0x17d, 0xbf, 0x2, 0x4ae, 0x4af, + 0x5, 0x193, 0xca, 0x2, 0x4af, 0x4b0, 0x5, 0x18d, 0xc7, 0x2, 0x4b0, 0x4b1, + 0x5, 0x1af, 0xd8, 0x2, 0x4b1, 0x4b2, 0x5, 0x185, 0xc3, 0x2, 0x4b2, 0x4b3, + 0x5, 0x183, 0xc2, 0x2, 0x4b3, 0xcc, 0x3, 0x2, 0x2, 0x2, 0x4b4, 0x4b5, + 0x5, 0x195, 0xcb, 0x2, 0x4b5, 0x4b6, 0x5, 0x17d, 0xbf, 0x2, 0x4b6, 0x4b7, + 0x5, 0x1ab, 0xd6, 0x2, 0x4b7, 0xce, 0x3, 0x2, 0x2, 0x2, 0x4b8, 0x4b9, + 0x5, 0x195, 0xcb, 0x2, 0x4b9, 0x4ba, 0x5, 0x185, 0xc3, 0x2, 0x4ba, 0x4bb, + 0x5, 0x19f, 0xd0, 0x2, 0x4bb, 0x4bc, 0x5, 0x189, 0xc5, 0x2, 0x4bc, 0x4bd, + 0x5, 0x185, 0xc3, 0x2, 0x4bd, 0x4be, 0x5, 0x1a1, 0xd1, 0x2, 0x4be, 0xd0, + 0x3, 0x2, 0x2, 0x2, 0x4bf, 0x4c0, 0x5, 0x195, 0xcb, 0x2, 0x4c0, 0x4c1, + 0x5, 0x18d, 0xc7, 0x2, 0x4c1, 0x4c2, 0x5, 0x197, 0xcc, 0x2, 0x4c2, 0xd2, + 0x3, 0x2, 0x2, 0x2, 0x4c3, 0x4c4, 0x5, 0x195, 0xcb, 0x2, 0x4c4, 0x4c5, + 0x5, 0x18d, 0xc7, 0x2, 0x4c5, 0x4c6, 0x5, 0x197, 0xcc, 0x2, 0x4c6, 0x4c7, + 0x5, 0x1a5, 0xd3, 0x2, 0x4c7, 0x4c8, 0x5, 0x1a3, 0xd2, 0x2, 0x4c8, 0x4c9, + 0x5, 0x185, 0xc3, 0x2, 0x4c9, 0xd4, 0x3, 0x2, 0x2, 0x2, 0x4ca, 0x4cb, + 0x5, 0x195, 0xcb, 0x2, 0x4cb, 0x4cc, 0x5, 0x199, 0xcd, 0x2, 0x4cc, 0x4cd, + 0x5, 0x183, 0xc2, 0x2, 0x4cd, 0x4ce, 0x5, 0x18d, 0xc7, 0x2, 0x4ce, 0x4cf, + 0x5, 0x187, 0xc4, 0x2, 0x4cf, 0x4d0, 0x5, 0x1ad, 0xd7, 0x2, 0x4d0, 0xd6, + 0x3, 0x2, 0x2, 0x2, 0x4d1, 0x4d2, 0x5, 0x195, 0xcb, 0x2, 0x4d2, 0x4d3, + 0x5, 0x199, 0xcd, 0x2, 0x4d3, 0x4d4, 0x5, 0x197, 0xcc, 0x2, 0x4d4, 0x4d5, + 0x5, 0x1a3, 0xd2, 0x2, 0x4d5, 0x4d6, 0x5, 0x18b, 0xc6, 0x2, 0x4d6, 0xd8, + 0x3, 0x2, 0x2, 0x2, 0x4d7, 0x4d8, 0x5, 0x195, 0xcb, 0x2, 0x4d8, 0x4d9, + 0x5, 0x199, 0xcd, 0x2, 0x4d9, 0x4da, 0x5, 0x1a7, 0xd4, 0x2, 0x4da, 0x4db, + 0x5, 0x185, 0xc3, 0x2, 0x4db, 0xda, 0x3, 0x2, 0x2, 0x2, 0x4dc, 0x4dd, + 0x5, 0x195, 0xcb, 0x2, 0x4dd, 0x4de, 0x5, 0x1a5, 0xd3, 0x2, 0x4de, 0x4df, + 0x5, 0x1a3, 0xd2, 0x2, 0x4df, 0x4e0, 0x5, 0x17d, 0xbf, 0x2, 0x4e0, 0x4e1, + 0x5, 0x1a3, 0xd2, 0x2, 0x4e1, 0x4e2, 0x5, 0x18d, 0xc7, 0x2, 0x4e2, 0x4e3, + 0x5, 0x199, 0xcd, 0x2, 0x4e3, 0x4e4, 0x5, 0x197, 0xcc, 0x2, 0x4e4, 0xdc, + 0x3, 0x2, 0x2, 0x2, 0x4e5, 0x4e6, 0x5, 0x197, 0xcc, 0x2, 0x4e6, 0x4e7, + 0x5, 0x17d, 0xbf, 0x2, 0x4e7, 0x4e8, 0x5, 0x197, 0xcc, 0x2, 0x4e8, 0xde, + 0x3, 0x2, 0x2, 0x2, 0x4e9, 0x4ea, 0x5, 0x197, 0xcc, 0x2, 0x4ea, 0x4eb, + 0x5, 0x199, 0xcd, 0x2, 0x4eb, 0xe0, 0x3, 0x2, 0x2, 0x2, 0x4ec, 0x4ed, + 0x5, 0x197, 0xcc, 0x2, 0x4ed, 0x4ee, 0x5, 0x199, 0xcd, 0x2, 0x4ee, 0x4ef, + 0x5, 0x1a3, 0xd2, 0x2, 0x4ef, 0xe2, 0x3, 0x2, 0x2, 0x2, 0x4f0, 0x4f1, + 0x5, 0x197, 0xcc, 0x2, 0x4f1, 0x4f2, 0x5, 0x1a5, 0xd3, 0x2, 0x4f2, 0x4f3, + 0x5, 0x193, 0xca, 0x2, 0x4f3, 0x4f4, 0x5, 0x193, 0xca, 0x2, 0x4f4, 0xe4, 0x3, 0x2, 0x2, 0x2, 0x4f5, 0x4f6, 0x5, 0x197, 0xcc, 0x2, 0x4f6, 0x4f7, - 0x5, 0x185, 0xc3, 0x2, 0x4f7, 0x4f8, 0x5, 0x185, 0xc3, 0x2, 0x4f8, 0x4f9, - 0x5, 0x19f, 0xd0, 0x2, 0x4f9, 0x4fa, 0x5, 0x183, 0xc2, 0x2, 0x4fa, 0x4fb, - 0x5, 0x1a1, 0xd1, 0x2, 0x4fb, 0xe6, 0x3, 0x2, 0x2, 0x2, 0x4fc, 0x4fd, - 0x5, 0x197, 0xcc, 0x2, 0x4fd, 0x4fe, 0x5, 0x195, 0xcb, 0x2, 0x4fe, 0xe8, - 0x3, 0x2, 0x2, 0x2, 0x4ff, 0x500, 0x5, 0x197, 0xcc, 0x2, 0x500, 0x501, - 0x5, 0x199, 0xcd, 0x2, 0x501, 0x502, 0x5, 0x1a1, 0xd1, 0x2, 0x502, 0x503, - 0x5, 0x18b, 0xc6, 0x2, 0x503, 0x504, 0x5, 0x193, 0xca, 0x2, 0x504, 0x505, - 0x5, 0x18b, 0xc6, 0x2, 0x505, 0x506, 0x5, 0x1ad, 0xd7, 0x2, 0x506, 0x507, - 0x5, 0x183, 0xc2, 0x2, 0x507, 0xea, 0x3, 0x2, 0x2, 0x2, 0x508, 0x509, - 0x5, 0x197, 0xcc, 0x2, 0x509, 0x50a, 0x5, 0x19d, 0xcf, 0x2, 0x50a, 0xec, - 0x3, 0x2, 0x2, 0x2, 0x50b, 0x50c, 0x5, 0x197, 0xcc, 0x2, 0x50c, 0x50d, - 0x5, 0x19d, 0xcf, 0x2, 0x50d, 0x50e, 0x5, 0x181, 0xc1, 0x2, 0x50e, 0x50f, - 0x5, 0x183, 0xc2, 0x2, 0x50f, 0x510, 0x5, 0x19d, 0xcf, 0x2, 0x510, 0xee, - 0x3, 0x2, 0x2, 0x2, 0x511, 0x512, 0x5, 0x197, 0xcc, 0x2, 0x512, 0x513, - 0x5, 0x1a3, 0xd2, 0x2, 0x513, 0x514, 0x5, 0x1a1, 0xd1, 0x2, 0x514, 0x515, - 0x5, 0x183, 0xc2, 0x2, 0x515, 0x516, 0x5, 0x19d, 0xcf, 0x2, 0x516, 0xf0, - 0x3, 0x2, 0x2, 0x2, 0x517, 0x518, 0x5, 0x197, 0xcc, 0x2, 0x518, 0x519, - 0x5, 0x1a3, 0xd2, 0x2, 0x519, 0x51a, 0x5, 0x1a1, 0xd1, 0x2, 0x51a, 0x51b, - 0x5, 0x185, 0xc3, 0x2, 0x51b, 0x51c, 0x5, 0x18b, 0xc6, 0x2, 0x51c, 0x51d, - 0x5, 0x191, 0xc9, 0x2, 0x51d, 0x51e, 0x5, 0x183, 0xc2, 0x2, 0x51e, 0xf2, - 0x3, 0x2, 0x2, 0x2, 0x51f, 0x520, 0x5, 0x199, 0xcd, 0x2, 0x520, 0x521, - 0x5, 0x17b, 0xbe, 0x2, 0x521, 0x522, 0x5, 0x19d, 0xcf, 0x2, 0x522, 0x523, - 0x5, 0x1a1, 0xd1, 0x2, 0x523, 0x524, 0x5, 0x18b, 0xc6, 0x2, 0x524, 0x525, - 0x5, 0x1a1, 0xd1, 0x2, 0x525, 0x526, 0x5, 0x18b, 0xc6, 0x2, 0x526, 0x527, - 0x5, 0x197, 0xcc, 0x2, 0x527, 0x528, 0x5, 0x195, 0xcb, 0x2, 0x528, 0xf4, - 0x3, 0x2, 0x2, 0x2, 0x529, 0x52a, 0x5, 0x199, 0xcd, 0x2, 0x52a, 0x52b, - 0x5, 0x197, 0xcc, 0x2, 0x52b, 0x52c, 0x5, 0x199, 0xcd, 0x2, 0x52c, 0x52d, - 0x5, 0x1a3, 0xd2, 0x2, 0x52d, 0x52e, 0x5, 0x191, 0xc9, 0x2, 0x52e, 0x52f, - 0x5, 0x17b, 0xbe, 0x2, 0x52f, 0x530, 0x5, 0x1a1, 0xd1, 0x2, 0x530, 0x531, - 0x5, 0x183, 0xc2, 0x2, 0x531, 0xf6, 0x3, 0x2, 0x2, 0x2, 0x532, 0x533, - 0x5, 0x199, 0xcd, 0x2, 0x533, 0x534, 0x5, 0x19d, 0xcf, 0x2, 0x534, 0x535, - 0x5, 0x183, 0xc2, 0x2, 0x535, 0x536, 0x5, 0x1a7, 0xd4, 0x2, 0x536, 0x537, - 0x5, 0x189, 0xc5, 0x2, 0x537, 0x538, 0x5, 0x183, 0xc2, 0x2, 0x538, 0x539, - 0x5, 0x19d, 0xcf, 0x2, 0x539, 0x53a, 0x5, 0x183, 0xc2, 0x2, 0x53a, 0xf8, - 0x3, 0x2, 0x2, 0x2, 0x53b, 0x53c, 0x5, 0x199, 0xcd, 0x2, 0x53c, 0x53d, - 0x5, 0x19d, 0xcf, 0x2, 0x53d, 0x53e, 0x5, 0x18b, 0xc6, 0x2, 0x53e, 0x53f, - 0x5, 0x193, 0xca, 0x2, 0x53f, 0x540, 0x5, 0x17b, 0xbe, 0x2, 0x540, 0x541, - 0x5, 0x19d, 0xcf, 0x2, 0x541, 0x542, 0x5, 0x1ab, 0xd6, 0x2, 0x542, 0xfa, - 0x3, 0x2, 0x2, 0x2, 0x543, 0x544, 0x5, 0x199, 0xcd, 0x2, 0x544, 0x545, - 0x5, 0x19d, 0xcf, 0x2, 0x545, 0x546, 0x5, 0x197, 0xcc, 0x2, 0x546, 0x547, - 0x5, 0x18d, 0xc7, 0x2, 0x547, 0x548, 0x5, 0x183, 0xc2, 0x2, 0x548, 0x549, - 0x5, 0x17f, 0xc0, 0x2, 0x549, 0x54a, 0x5, 0x1a1, 0xd1, 0x2, 0x54a, 0x54b, - 0x5, 0x18b, 0xc6, 0x2, 0x54b, 0x54c, 0x5, 0x197, 0xcc, 0x2, 0x54c, 0x54d, - 0x5, 0x195, 0xcb, 0x2, 0x54d, 0xfc, 0x3, 0x2, 0x2, 0x2, 0x54e, 0x54f, - 0x5, 0x19b, 0xce, 0x2, 0x54f, 0x550, 0x5, 0x1a3, 0xd2, 0x2, 0x550, 0x551, - 0x5, 0x17b, 0xbe, 0x2, 0x551, 0x552, 0x5, 0x19d, 0xcf, 0x2, 0x552, 0x553, - 0x5, 0x1a1, 0xd1, 0x2, 0x553, 0x554, 0x5, 0x183, 0xc2, 0x2, 0x554, 0x555, - 0x5, 0x19d, 0xcf, 0x2, 0x555, 0xfe, 0x3, 0x2, 0x2, 0x2, 0x556, 0x557, - 0x5, 0x19d, 0xcf, 0x2, 0x557, 0x558, 0x5, 0x17b, 0xbe, 0x2, 0x558, 0x559, - 0x5, 0x195, 0xcb, 0x2, 0x559, 0x55a, 0x5, 0x187, 0xc4, 0x2, 0x55a, 0x55b, - 0x5, 0x183, 0xc2, 0x2, 0x55b, 0x100, 0x3, 0x2, 0x2, 0x2, 0x55c, 0x55d, - 0x5, 0x19d, 0xcf, 0x2, 0x55d, 0x55e, 0x5, 0x183, 0xc2, 0x2, 0x55e, 0x55f, - 0x5, 0x191, 0xc9, 0x2, 0x55f, 0x560, 0x5, 0x197, 0xcc, 0x2, 0x560, 0x561, - 0x5, 0x17b, 0xbe, 0x2, 0x561, 0x562, 0x5, 0x181, 0xc1, 0x2, 0x562, 0x102, - 0x3, 0x2, 0x2, 0x2, 0x563, 0x564, 0x5, 0x19d, 0xcf, 0x2, 0x564, 0x565, - 0x5, 0x183, 0xc2, 0x2, 0x565, 0x566, 0x5, 0x193, 0xca, 0x2, 0x566, 0x567, - 0x5, 0x197, 0xcc, 0x2, 0x567, 0x568, 0x5, 0x1a5, 0xd3, 0x2, 0x568, 0x569, - 0x5, 0x183, 0xc2, 0x2, 0x569, 0x104, 0x3, 0x2, 0x2, 0x2, 0x56a, 0x56b, - 0x5, 0x19d, 0xcf, 0x2, 0x56b, 0x56c, 0x5, 0x183, 0xc2, 0x2, 0x56c, 0x56d, - 0x5, 0x195, 0xcb, 0x2, 0x56d, 0x56e, 0x5, 0x17b, 0xbe, 0x2, 0x56e, 0x56f, - 0x5, 0x193, 0xca, 0x2, 0x56f, 0x570, 0x5, 0x183, 0xc2, 0x2, 0x570, 0x106, - 0x3, 0x2, 0x2, 0x2, 0x571, 0x572, 0x5, 0x19d, 0xcf, 0x2, 0x572, 0x573, - 0x5, 0x183, 0xc2, 0x2, 0x573, 0x574, 0x5, 0x199, 0xcd, 0x2, 0x574, 0x575, - 0x5, 0x191, 0xc9, 0x2, 0x575, 0x576, 0x5, 0x17b, 0xbe, 0x2, 0x576, 0x577, - 0x5, 0x17f, 0xc0, 0x2, 0x577, 0x578, 0x5, 0x183, 0xc2, 0x2, 0x578, 0x108, - 0x3, 0x2, 0x2, 0x2, 0x579, 0x57a, 0x5, 0x19d, 0xcf, 0x2, 0x57a, 0x57b, - 0x5, 0x183, 0xc2, 0x2, 0x57b, 0x57c, 0x5, 0x199, 0xcd, 0x2, 0x57c, 0x57d, - 0x5, 0x191, 0xc9, 0x2, 0x57d, 0x57e, 0x5, 0x18b, 0xc6, 0x2, 0x57e, 0x57f, - 0x5, 0x17f, 0xc0, 0x2, 0x57f, 0x580, 0x5, 0x17b, 0xbe, 0x2, 0x580, 0x10a, - 0x3, 0x2, 0x2, 0x2, 0x581, 0x582, 0x5, 0x19d, 0xcf, 0x2, 0x582, 0x583, - 0x5, 0x183, 0xc2, 0x2, 0x583, 0x584, 0x5, 0x199, 0xcd, 0x2, 0x584, 0x585, - 0x5, 0x191, 0xc9, 0x2, 0x585, 0x586, 0x5, 0x18b, 0xc6, 0x2, 0x586, 0x587, - 0x5, 0x17f, 0xc0, 0x2, 0x587, 0x588, 0x5, 0x17b, 0xbe, 0x2, 0x588, 0x589, - 0x5, 0x1a1, 0xd1, 0x2, 0x589, 0x58a, 0x5, 0x183, 0xc2, 0x2, 0x58a, 0x58b, - 0x5, 0x181, 0xc1, 0x2, 0x58b, 0x10c, 0x3, 0x2, 0x2, 0x2, 0x58c, 0x58d, - 0x5, 0x19d, 0xcf, 0x2, 0x58d, 0x58e, 0x5, 0x18b, 0xc6, 0x2, 0x58e, 0x58f, - 0x5, 0x187, 0xc4, 0x2, 0x58f, 0x590, 0x5, 0x189, 0xc5, 0x2, 0x590, 0x591, - 0x5, 0x1a1, 0xd1, 0x2, 0x591, 0x10e, 0x3, 0x2, 0x2, 0x2, 0x592, 0x593, - 0x5, 0x19d, 0xcf, 0x2, 0x593, 0x594, 0x5, 0x197, 0xcc, 0x2, 0x594, 0x595, - 0x5, 0x191, 0xc9, 0x2, 0x595, 0x596, 0x5, 0x191, 0xc9, 0x2, 0x596, 0x597, - 0x5, 0x1a3, 0xd2, 0x2, 0x597, 0x598, 0x5, 0x199, 0xcd, 0x2, 0x598, 0x110, - 0x3, 0x2, 0x2, 0x2, 0x599, 0x59a, 0x5, 0x19f, 0xd0, 0x2, 0x59a, 0x59b, - 0x5, 0x17b, 0xbe, 0x2, 0x59b, 0x59c, 0x5, 0x193, 0xca, 0x2, 0x59c, 0x59d, - 0x5, 0x199, 0xcd, 0x2, 0x59d, 0x59e, 0x5, 0x191, 0xc9, 0x2, 0x59e, 0x59f, - 0x5, 0x183, 0xc2, 0x2, 0x59f, 0x112, 0x3, 0x2, 0x2, 0x2, 0x5a0, 0x5a1, - 0x5, 0x19f, 0xd0, 0x2, 0x5a1, 0x5a2, 0x5, 0x183, 0xc2, 0x2, 0x5a2, 0x5a3, - 0x5, 0x17f, 0xc0, 0x2, 0x5a3, 0x5a4, 0x5, 0x197, 0xcc, 0x2, 0x5a4, 0x5a5, - 0x5, 0x195, 0xcb, 0x2, 0x5a5, 0x5a6, 0x5, 0x181, 0xc1, 0x2, 0x5a6, 0x114, - 0x3, 0x2, 0x2, 0x2, 0x5a7, 0x5a8, 0x5, 0x19f, 0xd0, 0x2, 0x5a8, 0x5a9, - 0x5, 0x183, 0xc2, 0x2, 0x5a9, 0x5aa, 0x5, 0x191, 0xc9, 0x2, 0x5aa, 0x5ab, - 0x5, 0x183, 0xc2, 0x2, 0x5ab, 0x5ac, 0x5, 0x17f, 0xc0, 0x2, 0x5ac, 0x5ad, - 0x5, 0x1a1, 0xd1, 0x2, 0x5ad, 0x116, 0x3, 0x2, 0x2, 0x2, 0x5ae, 0x5af, - 0x5, 0x19f, 0xd0, 0x2, 0x5af, 0x5b0, 0x5, 0x183, 0xc2, 0x2, 0x5b0, 0x5b1, - 0x5, 0x193, 0xca, 0x2, 0x5b1, 0x5b2, 0x5, 0x18b, 0xc6, 0x2, 0x5b2, 0x118, - 0x3, 0x2, 0x2, 0x2, 0x5b3, 0x5b4, 0x5, 0x19f, 0xd0, 0x2, 0x5b4, 0x5b5, - 0x5, 0x183, 0xc2, 0x2, 0x5b5, 0x5b6, 0x5, 0x195, 0xcb, 0x2, 0x5b6, 0x5b7, - 0x5, 0x181, 0xc1, 0x2, 0x5b7, 0x5b8, 0x5, 0x19f, 0xd0, 0x2, 0x5b8, 0x11a, - 0x3, 0x2, 0x2, 0x2, 0x5b9, 0x5ba, 0x5, 0x19f, 0xd0, 0x2, 0x5ba, 0x5bb, - 0x5, 0x183, 0xc2, 0x2, 0x5bb, 0x5bc, 0x5, 0x1a1, 0xd1, 0x2, 0x5bc, 0x11c, - 0x3, 0x2, 0x2, 0x2, 0x5bd, 0x5be, 0x5, 0x19f, 0xd0, 0x2, 0x5be, 0x5bf, - 0x5, 0x183, 0xc2, 0x2, 0x5bf, 0x5c0, 0x5, 0x1a1, 0xd1, 0x2, 0x5c0, 0x5c1, - 0x5, 0x1a1, 0xd1, 0x2, 0x5c1, 0x5c2, 0x5, 0x18b, 0xc6, 0x2, 0x5c2, 0x5c3, - 0x5, 0x195, 0xcb, 0x2, 0x5c3, 0x5c4, 0x5, 0x187, 0xc4, 0x2, 0x5c4, 0x5c5, - 0x5, 0x19f, 0xd0, 0x2, 0x5c5, 0x11e, 0x3, 0x2, 0x2, 0x2, 0x5c6, 0x5c7, - 0x5, 0x19f, 0xd0, 0x2, 0x5c7, 0x5c8, 0x5, 0x189, 0xc5, 0x2, 0x5c8, 0x5c9, - 0x5, 0x197, 0xcc, 0x2, 0x5c9, 0x5ca, 0x5, 0x1a7, 0xd4, 0x2, 0x5ca, 0x120, - 0x3, 0x2, 0x2, 0x2, 0x5cb, 0x5cc, 0x5, 0x19f, 0xd0, 0x2, 0x5cc, 0x5cd, - 0x5, 0x197, 0xcc, 0x2, 0x5cd, 0x5ce, 0x5, 0x1a3, 0xd2, 0x2, 0x5ce, 0x5cf, - 0x5, 0x19d, 0xcf, 0x2, 0x5cf, 0x5d0, 0x5, 0x17f, 0xc0, 0x2, 0x5d0, 0x5d1, - 0x5, 0x183, 0xc2, 0x2, 0x5d1, 0x122, 0x3, 0x2, 0x2, 0x2, 0x5d2, 0x5d3, - 0x5, 0x19f, 0xd0, 0x2, 0x5d3, 0x5d4, 0x5, 0x1a1, 0xd1, 0x2, 0x5d4, 0x5d5, - 0x5, 0x17b, 0xbe, 0x2, 0x5d5, 0x5d6, 0x5, 0x19d, 0xcf, 0x2, 0x5d6, 0x5d7, - 0x5, 0x1a1, 0xd1, 0x2, 0x5d7, 0x124, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5d9, - 0x5, 0x19f, 0xd0, 0x2, 0x5d9, 0x5da, 0x5, 0x1a1, 0xd1, 0x2, 0x5da, 0x5db, - 0x5, 0x197, 0xcc, 0x2, 0x5db, 0x5dc, 0x5, 0x199, 0xcd, 0x2, 0x5dc, 0x126, - 0x3, 0x2, 0x2, 0x2, 0x5dd, 0x5de, 0x5, 0x19f, 0xd0, 0x2, 0x5de, 0x5df, - 0x5, 0x1a3, 0xd2, 0x2, 0x5df, 0x5e0, 0x5, 0x17d, 0xbf, 0x2, 0x5e0, 0x5e1, - 0x5, 0x19f, 0xd0, 0x2, 0x5e1, 0x5e2, 0x5, 0x1a1, 0xd1, 0x2, 0x5e2, 0x5e3, - 0x5, 0x19d, 0xcf, 0x2, 0x5e3, 0x5e4, 0x5, 0x18b, 0xc6, 0x2, 0x5e4, 0x5e5, - 0x5, 0x195, 0xcb, 0x2, 0x5e5, 0x5e6, 0x5, 0x187, 0xc4, 0x2, 0x5e6, 0x128, - 0x3, 0x2, 0x2, 0x2, 0x5e7, 0x5e8, 0x5, 0x19f, 0xd0, 0x2, 0x5e8, 0x5e9, - 0x5, 0x1ab, 0xd6, 0x2, 0x5e9, 0x5ea, 0x5, 0x195, 0xcb, 0x2, 0x5ea, 0x5eb, - 0x5, 0x17f, 0xc0, 0x2, 0x5eb, 0x12a, 0x3, 0x2, 0x2, 0x2, 0x5ec, 0x5ed, - 0x5, 0x19f, 0xd0, 0x2, 0x5ed, 0x5ee, 0x5, 0x1ab, 0xd6, 0x2, 0x5ee, 0x5ef, - 0x5, 0x195, 0xcb, 0x2, 0x5ef, 0x5f0, 0x5, 0x1a1, 0xd1, 0x2, 0x5f0, 0x5f1, - 0x5, 0x17b, 0xbe, 0x2, 0x5f1, 0x5f2, 0x5, 0x1a9, 0xd5, 0x2, 0x5f2, 0x12c, - 0x3, 0x2, 0x2, 0x2, 0x5f3, 0x5f4, 0x5, 0x19f, 0xd0, 0x2, 0x5f4, 0x5f5, - 0x5, 0x1ab, 0xd6, 0x2, 0x5f5, 0x5f6, 0x5, 0x19f, 0xd0, 0x2, 0x5f6, 0x5f7, - 0x5, 0x1a1, 0xd1, 0x2, 0x5f7, 0x5f8, 0x5, 0x183, 0xc2, 0x2, 0x5f8, 0x5f9, - 0x5, 0x193, 0xca, 0x2, 0x5f9, 0x12e, 0x3, 0x2, 0x2, 0x2, 0x5fa, 0x5fb, - 0x5, 0x1a1, 0xd1, 0x2, 0x5fb, 0x5fc, 0x5, 0x17b, 0xbe, 0x2, 0x5fc, 0x5fd, - 0x5, 0x17d, 0xbf, 0x2, 0x5fd, 0x5fe, 0x5, 0x191, 0xc9, 0x2, 0x5fe, 0x5ff, - 0x5, 0x183, 0xc2, 0x2, 0x5ff, 0x130, 0x3, 0x2, 0x2, 0x2, 0x600, 0x601, - 0x5, 0x1a1, 0xd1, 0x2, 0x601, 0x602, 0x5, 0x17b, 0xbe, 0x2, 0x602, 0x603, - 0x5, 0x17d, 0xbf, 0x2, 0x603, 0x604, 0x5, 0x191, 0xc9, 0x2, 0x604, 0x605, - 0x5, 0x183, 0xc2, 0x2, 0x605, 0x606, 0x5, 0x19f, 0xd0, 0x2, 0x606, 0x132, - 0x3, 0x2, 0x2, 0x2, 0x607, 0x608, 0x5, 0x1a1, 0xd1, 0x2, 0x608, 0x609, - 0x5, 0x183, 0xc2, 0x2, 0x609, 0x60a, 0x5, 0x193, 0xca, 0x2, 0x60a, 0x60b, - 0x5, 0x199, 0xcd, 0x2, 0x60b, 0x60c, 0x5, 0x197, 0xcc, 0x2, 0x60c, 0x60d, - 0x5, 0x19d, 0xcf, 0x2, 0x60d, 0x60e, 0x5, 0x17b, 0xbe, 0x2, 0x60e, 0x60f, - 0x5, 0x19d, 0xcf, 0x2, 0x60f, 0x610, 0x5, 0x1ab, 0xd6, 0x2, 0x610, 0x134, - 0x3, 0x2, 0x2, 0x2, 0x611, 0x612, 0x5, 0x1a1, 0xd1, 0x2, 0x612, 0x613, - 0x5, 0x183, 0xc2, 0x2, 0x613, 0x614, 0x5, 0x19f, 0xd0, 0x2, 0x614, 0x615, - 0x5, 0x1a1, 0xd1, 0x2, 0x615, 0x136, 0x3, 0x2, 0x2, 0x2, 0x616, 0x617, - 0x5, 0x1a1, 0xd1, 0x2, 0x617, 0x618, 0x5, 0x189, 0xc5, 0x2, 0x618, 0x619, - 0x5, 0x183, 0xc2, 0x2, 0x619, 0x61a, 0x5, 0x195, 0xcb, 0x2, 0x61a, 0x138, - 0x3, 0x2, 0x2, 0x2, 0x61b, 0x61c, 0x5, 0x1a1, 0xd1, 0x2, 0x61c, 0x61d, - 0x5, 0x18b, 0xc6, 0x2, 0x61d, 0x61e, 0x5, 0x183, 0xc2, 0x2, 0x61e, 0x61f, - 0x5, 0x19f, 0xd0, 0x2, 0x61f, 0x13a, 0x3, 0x2, 0x2, 0x2, 0x620, 0x621, - 0x5, 0x1a1, 0xd1, 0x2, 0x621, 0x622, 0x5, 0x18b, 0xc6, 0x2, 0x622, 0x623, - 0x5, 0x193, 0xca, 0x2, 0x623, 0x624, 0x5, 0x183, 0xc2, 0x2, 0x624, 0x625, - 0x5, 0x197, 0xcc, 0x2, 0x625, 0x626, 0x5, 0x1a3, 0xd2, 0x2, 0x626, 0x627, - 0x5, 0x1a1, 0xd1, 0x2, 0x627, 0x13c, 0x3, 0x2, 0x2, 0x2, 0x628, 0x629, - 0x5, 0x1a1, 0xd1, 0x2, 0x629, 0x62a, 0x5, 0x18b, 0xc6, 0x2, 0x62a, 0x62b, - 0x5, 0x193, 0xca, 0x2, 0x62b, 0x62c, 0x5, 0x183, 0xc2, 0x2, 0x62c, 0x62d, - 0x5, 0x19f, 0xd0, 0x2, 0x62d, 0x62e, 0x5, 0x1a1, 0xd1, 0x2, 0x62e, 0x62f, - 0x5, 0x17b, 0xbe, 0x2, 0x62f, 0x630, 0x5, 0x193, 0xca, 0x2, 0x630, 0x631, - 0x5, 0x199, 0xcd, 0x2, 0x631, 0x13e, 0x3, 0x2, 0x2, 0x2, 0x632, 0x633, - 0x5, 0x1a1, 0xd1, 0x2, 0x633, 0x634, 0x5, 0x197, 0xcc, 0x2, 0x634, 0x140, - 0x3, 0x2, 0x2, 0x2, 0x635, 0x636, 0x5, 0x1a1, 0xd1, 0x2, 0x636, 0x637, - 0x5, 0x197, 0xcc, 0x2, 0x637, 0x638, 0x5, 0x199, 0xcd, 0x2, 0x638, 0x142, - 0x3, 0x2, 0x2, 0x2, 0x639, 0x63a, 0x5, 0x1a1, 0xd1, 0x2, 0x63a, 0x63b, - 0x5, 0x197, 0xcc, 0x2, 0x63b, 0x63c, 0x5, 0x1a1, 0xd1, 0x2, 0x63c, 0x63d, - 0x5, 0x17b, 0xbe, 0x2, 0x63d, 0x63e, 0x5, 0x191, 0xc9, 0x2, 0x63e, 0x63f, - 0x5, 0x19f, 0xd0, 0x2, 0x63f, 0x144, 0x3, 0x2, 0x2, 0x2, 0x640, 0x641, - 0x5, 0x1a1, 0xd1, 0x2, 0x641, 0x642, 0x5, 0x19d, 0xcf, 0x2, 0x642, 0x643, - 0x5, 0x17b, 0xbe, 0x2, 0x643, 0x644, 0x5, 0x18b, 0xc6, 0x2, 0x644, 0x645, - 0x5, 0x191, 0xc9, 0x2, 0x645, 0x646, 0x5, 0x18b, 0xc6, 0x2, 0x646, 0x647, - 0x5, 0x195, 0xcb, 0x2, 0x647, 0x648, 0x5, 0x187, 0xc4, 0x2, 0x648, 0x146, - 0x3, 0x2, 0x2, 0x2, 0x649, 0x64a, 0x5, 0x1a1, 0xd1, 0x2, 0x64a, 0x64b, - 0x5, 0x19d, 0xcf, 0x2, 0x64b, 0x64c, 0x5, 0x18b, 0xc6, 0x2, 0x64c, 0x64d, - 0x5, 0x193, 0xca, 0x2, 0x64d, 0x148, 0x3, 0x2, 0x2, 0x2, 0x64e, 0x64f, - 0x5, 0x1a1, 0xd1, 0x2, 0x64f, 0x650, 0x5, 0x19d, 0xcf, 0x2, 0x650, 0x651, - 0x5, 0x1a3, 0xd2, 0x2, 0x651, 0x652, 0x5, 0x195, 0xcb, 0x2, 0x652, 0x653, - 0x5, 0x17f, 0xc0, 0x2, 0x653, 0x654, 0x5, 0x17b, 0xbe, 0x2, 0x654, 0x655, - 0x5, 0x1a1, 0xd1, 0x2, 0x655, 0x656, 0x5, 0x183, 0xc2, 0x2, 0x656, 0x14a, - 0x3, 0x2, 0x2, 0x2, 0x657, 0x658, 0x5, 0x1a1, 0xd1, 0x2, 0x658, 0x659, - 0x5, 0x1a1, 0xd1, 0x2, 0x659, 0x65a, 0x5, 0x191, 0xc9, 0x2, 0x65a, 0x14c, - 0x3, 0x2, 0x2, 0x2, 0x65b, 0x65c, 0x5, 0x1a1, 0xd1, 0x2, 0x65c, 0x65d, - 0x5, 0x1ab, 0xd6, 0x2, 0x65d, 0x65e, 0x5, 0x199, 0xcd, 0x2, 0x65e, 0x65f, - 0x5, 0x183, 0xc2, 0x2, 0x65f, 0x14e, 0x3, 0x2, 0x2, 0x2, 0x660, 0x661, - 0x5, 0x1a3, 0xd2, 0x2, 0x661, 0x662, 0x5, 0x195, 0xcb, 0x2, 0x662, 0x663, - 0x5, 0x18b, 0xc6, 0x2, 0x663, 0x664, 0x5, 0x197, 0xcc, 0x2, 0x664, 0x665, - 0x5, 0x195, 0xcb, 0x2, 0x665, 0x150, 0x3, 0x2, 0x2, 0x2, 0x666, 0x667, - 0x5, 0x1a3, 0xd2, 0x2, 0x667, 0x668, 0x5, 0x199, 0xcd, 0x2, 0x668, 0x669, - 0x5, 0x181, 0xc1, 0x2, 0x669, 0x66a, 0x5, 0x17b, 0xbe, 0x2, 0x66a, 0x66b, - 0x5, 0x1a1, 0xd1, 0x2, 0x66b, 0x66c, 0x5, 0x183, 0xc2, 0x2, 0x66c, 0x152, - 0x3, 0x2, 0x2, 0x2, 0x66d, 0x66e, 0x5, 0x1a3, 0xd2, 0x2, 0x66e, 0x66f, - 0x5, 0x19f, 0xd0, 0x2, 0x66f, 0x670, 0x5, 0x183, 0xc2, 0x2, 0x670, 0x154, - 0x3, 0x2, 0x2, 0x2, 0x671, 0x672, 0x5, 0x1a3, 0xd2, 0x2, 0x672, 0x673, - 0x5, 0x19f, 0xd0, 0x2, 0x673, 0x674, 0x5, 0x18b, 0xc6, 0x2, 0x674, 0x675, - 0x5, 0x195, 0xcb, 0x2, 0x675, 0x676, 0x5, 0x187, 0xc4, 0x2, 0x676, 0x156, - 0x3, 0x2, 0x2, 0x2, 0x677, 0x678, 0x5, 0x1a3, 0xd2, 0x2, 0x678, 0x679, - 0x5, 0x1a3, 0xd2, 0x2, 0x679, 0x67a, 0x5, 0x18b, 0xc6, 0x2, 0x67a, 0x67b, - 0x5, 0x181, 0xc1, 0x2, 0x67b, 0x158, 0x3, 0x2, 0x2, 0x2, 0x67c, 0x67d, - 0x5, 0x1a5, 0xd3, 0x2, 0x67d, 0x67e, 0x5, 0x17b, 0xbe, 0x2, 0x67e, 0x67f, - 0x5, 0x191, 0xc9, 0x2, 0x67f, 0x680, 0x5, 0x1a3, 0xd2, 0x2, 0x680, 0x681, - 0x5, 0x183, 0xc2, 0x2, 0x681, 0x682, 0x5, 0x19f, 0xd0, 0x2, 0x682, 0x15a, - 0x3, 0x2, 0x2, 0x2, 0x683, 0x684, 0x5, 0x1a5, 0xd3, 0x2, 0x684, 0x685, - 0x5, 0x18b, 0xc6, 0x2, 0x685, 0x686, 0x5, 0x183, 0xc2, 0x2, 0x686, 0x687, - 0x5, 0x1a7, 0xd4, 0x2, 0x687, 0x15c, 0x3, 0x2, 0x2, 0x2, 0x688, 0x689, - 0x5, 0x1a5, 0xd3, 0x2, 0x689, 0x68a, 0x5, 0x197, 0xcc, 0x2, 0x68a, 0x68b, - 0x5, 0x191, 0xc9, 0x2, 0x68b, 0x68c, 0x5, 0x1a3, 0xd2, 0x2, 0x68c, 0x68d, - 0x5, 0x193, 0xca, 0x2, 0x68d, 0x68e, 0x5, 0x183, 0xc2, 0x2, 0x68e, 0x15e, - 0x3, 0x2, 0x2, 0x2, 0x68f, 0x690, 0x5, 0x1a7, 0xd4, 0x2, 0x690, 0x691, - 0x5, 0x17b, 0xbe, 0x2, 0x691, 0x692, 0x5, 0x1a1, 0xd1, 0x2, 0x692, 0x693, - 0x5, 0x17f, 0xc0, 0x2, 0x693, 0x694, 0x5, 0x189, 0xc5, 0x2, 0x694, 0x160, - 0x3, 0x2, 0x2, 0x2, 0x695, 0x696, 0x5, 0x1a7, 0xd4, 0x2, 0x696, 0x697, - 0x5, 0x183, 0xc2, 0x2, 0x697, 0x698, 0x5, 0x183, 0xc2, 0x2, 0x698, 0x699, - 0x5, 0x18f, 0xc8, 0x2, 0x699, 0x162, 0x3, 0x2, 0x2, 0x2, 0x69a, 0x69b, - 0x5, 0x1a7, 0xd4, 0x2, 0x69b, 0x69c, 0x5, 0x189, 0xc5, 0x2, 0x69c, 0x69d, - 0x5, 0x183, 0xc2, 0x2, 0x69d, 0x69e, 0x5, 0x195, 0xcb, 0x2, 0x69e, 0x164, - 0x3, 0x2, 0x2, 0x2, 0x69f, 0x6a0, 0x5, 0x1a7, 0xd4, 0x2, 0x6a0, 0x6a1, - 0x5, 0x189, 0xc5, 0x2, 0x6a1, 0x6a2, 0x5, 0x183, 0xc2, 0x2, 0x6a2, 0x6a3, - 0x5, 0x19d, 0xcf, 0x2, 0x6a3, 0x6a4, 0x5, 0x183, 0xc2, 0x2, 0x6a4, 0x166, - 0x3, 0x2, 0x2, 0x2, 0x6a5, 0x6a6, 0x5, 0x1a7, 0xd4, 0x2, 0x6a6, 0x6a7, - 0x5, 0x18b, 0xc6, 0x2, 0x6a7, 0x6a8, 0x5, 0x1a1, 0xd1, 0x2, 0x6a8, 0x6a9, - 0x5, 0x189, 0xc5, 0x2, 0x6a9, 0x168, 0x3, 0x2, 0x2, 0x2, 0x6aa, 0x6ab, - 0x5, 0x1ab, 0xd6, 0x2, 0x6ab, 0x6ac, 0x5, 0x183, 0xc2, 0x2, 0x6ac, 0x6ad, - 0x5, 0x17b, 0xbe, 0x2, 0x6ad, 0x6ae, 0x5, 0x19d, 0xcf, 0x2, 0x6ae, 0x6b5, - 0x3, 0x2, 0x2, 0x2, 0x6af, 0x6b0, 0x5, 0x1ab, 0xd6, 0x2, 0x6b0, 0x6b1, - 0x5, 0x1ab, 0xd6, 0x2, 0x6b1, 0x6b2, 0x5, 0x1ab, 0xd6, 0x2, 0x6b2, 0x6b3, - 0x5, 0x1ab, 0xd6, 0x2, 0x6b3, 0x6b5, 0x3, 0x2, 0x2, 0x2, 0x6b4, 0x6aa, - 0x3, 0x2, 0x2, 0x2, 0x6b4, 0x6af, 0x3, 0x2, 0x2, 0x2, 0x6b5, 0x16a, - 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6b7, 0x7, 0x68, 0x2, 0x2, 0x6b7, 0x6b8, - 0x7, 0x63, 0x2, 0x2, 0x6b8, 0x6b9, 0x7, 0x6e, 0x2, 0x2, 0x6b9, 0x6ba, - 0x7, 0x75, 0x2, 0x2, 0x6ba, 0x6bb, 0x7, 0x67, 0x2, 0x2, 0x6bb, 0x16c, - 0x3, 0x2, 0x2, 0x2, 0x6bc, 0x6bd, 0x7, 0x76, 0x2, 0x2, 0x6bd, 0x6be, - 0x7, 0x74, 0x2, 0x2, 0x6be, 0x6bf, 0x7, 0x77, 0x2, 0x2, 0x6bf, 0x6c0, - 0x7, 0x67, 0x2, 0x2, 0x6c0, 0x16e, 0x3, 0x2, 0x2, 0x2, 0x6c1, 0x6c4, - 0x5, 0x1af, 0xd8, 0x2, 0x6c2, 0x6c4, 0x5, 0x1f1, 0xf9, 0x2, 0x6c3, 0x6c1, - 0x3, 0x2, 0x2, 0x2, 0x6c3, 0x6c2, 0x3, 0x2, 0x2, 0x2, 0x6c4, 0x6ca, - 0x3, 0x2, 0x2, 0x2, 0x6c5, 0x6c9, 0x5, 0x1af, 0xd8, 0x2, 0x6c6, 0x6c9, - 0x5, 0x1f1, 0xf9, 0x2, 0x6c7, 0x6c9, 0x5, 0x1b3, 0xda, 0x2, 0x6c8, 0x6c5, - 0x3, 0x2, 0x2, 0x2, 0x6c8, 0x6c6, 0x3, 0x2, 0x2, 0x2, 0x6c8, 0x6c7, - 0x3, 0x2, 0x2, 0x2, 0x6c9, 0x6cc, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6c8, - 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6cb, 0x3, 0x2, 0x2, 0x2, 0x6cb, 0x6ec, - 0x3, 0x2, 0x2, 0x2, 0x6cc, 0x6ca, 0x3, 0x2, 0x2, 0x2, 0x6cd, 0x6d7, - 0x5, 0x1bb, 0xde, 0x2, 0x6ce, 0x6d6, 0xa, 0x2, 0x2, 0x2, 0x6cf, 0x6d0, - 0x5, 0x1bd, 0xdf, 0x2, 0x6d0, 0x6d1, 0xb, 0x2, 0x2, 0x2, 0x6d1, 0x6d6, - 0x3, 0x2, 0x2, 0x2, 0x6d2, 0x6d3, 0x5, 0x1bb, 0xde, 0x2, 0x6d3, 0x6d4, - 0x5, 0x1bb, 0xde, 0x2, 0x6d4, 0x6d6, 0x3, 0x2, 0x2, 0x2, 0x6d5, 0x6ce, - 0x3, 0x2, 0x2, 0x2, 0x6d5, 0x6cf, 0x3, 0x2, 0x2, 0x2, 0x6d5, 0x6d2, - 0x3, 0x2, 0x2, 0x2, 0x6d6, 0x6d9, 0x3, 0x2, 0x2, 0x2, 0x6d7, 0x6d5, - 0x3, 0x2, 0x2, 0x2, 0x6d7, 0x6d8, 0x3, 0x2, 0x2, 0x2, 0x6d8, 0x6da, - 0x3, 0x2, 0x2, 0x2, 0x6d9, 0x6d7, 0x3, 0x2, 0x2, 0x2, 0x6da, 0x6db, - 0x5, 0x1bb, 0xde, 0x2, 0x6db, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6dc, 0x6e6, - 0x5, 0x1e3, 0xf2, 0x2, 0x6dd, 0x6e5, 0xa, 0x3, 0x2, 0x2, 0x6de, 0x6df, - 0x5, 0x1bd, 0xdf, 0x2, 0x6df, 0x6e0, 0xb, 0x2, 0x2, 0x2, 0x6e0, 0x6e5, - 0x3, 0x2, 0x2, 0x2, 0x6e1, 0x6e2, 0x5, 0x1e3, 0xf2, 0x2, 0x6e2, 0x6e3, - 0x5, 0x1e3, 0xf2, 0x2, 0x6e3, 0x6e5, 0x3, 0x2, 0x2, 0x2, 0x6e4, 0x6dd, - 0x3, 0x2, 0x2, 0x2, 0x6e4, 0x6de, 0x3, 0x2, 0x2, 0x2, 0x6e4, 0x6e1, - 0x3, 0x2, 0x2, 0x2, 0x6e5, 0x6e8, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6e4, - 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6e7, 0x3, 0x2, 0x2, 0x2, 0x6e7, 0x6e9, - 0x3, 0x2, 0x2, 0x2, 0x6e8, 0x6e6, 0x3, 0x2, 0x2, 0x2, 0x6e9, 0x6ea, - 0x5, 0x1e3, 0xf2, 0x2, 0x6ea, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6eb, 0x6c3, - 0x3, 0x2, 0x2, 0x2, 0x6eb, 0x6cd, 0x3, 0x2, 0x2, 0x2, 0x6eb, 0x6dc, - 0x3, 0x2, 0x2, 0x2, 0x6ec, 0x170, 0x3, 0x2, 0x2, 0x2, 0x6ed, 0x6ee, - 0x5, 0x177, 0xbc, 0x2, 0x6ee, 0x6f2, 0x5, 0x1c7, 0xe4, 0x2, 0x6ef, 0x6f1, - 0x5, 0x1b5, 0xdb, 0x2, 0x6f0, 0x6ef, 0x3, 0x2, 0x2, 0x2, 0x6f1, 0x6f4, - 0x3, 0x2, 0x2, 0x2, 0x6f2, 0x6f0, 0x3, 0x2, 0x2, 0x2, 0x6f2, 0x6f3, - 0x3, 0x2, 0x2, 0x2, 0x6f3, 0x6f7, 0x3, 0x2, 0x2, 0x2, 0x6f4, 0x6f2, - 0x3, 0x2, 0x2, 0x2, 0x6f5, 0x6f8, 0x5, 0x199, 0xcd, 0x2, 0x6f6, 0x6f8, - 0x5, 0x183, 0xc2, 0x2, 0x6f7, 0x6f5, 0x3, 0x2, 0x2, 0x2, 0x6f7, 0x6f6, - 0x3, 0x2, 0x2, 0x2, 0x6f8, 0x6fb, 0x3, 0x2, 0x2, 0x2, 0x6f9, 0x6fc, - 0x5, 0x1df, 0xf0, 0x2, 0x6fa, 0x6fc, 0x5, 0x1c5, 0xe3, 0x2, 0x6fb, 0x6f9, - 0x3, 0x2, 0x2, 0x2, 0x6fb, 0x6fa, 0x3, 0x2, 0x2, 0x2, 0x6fb, 0x6fc, - 0x3, 0x2, 0x2, 0x2, 0x6fc, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x6fd, 0x6ff, - 0x5, 0x1b3, 0xda, 0x2, 0x6fe, 0x6fd, 0x3, 0x2, 0x2, 0x2, 0x6ff, 0x700, - 0x3, 0x2, 0x2, 0x2, 0x700, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x700, 0x701, - 0x3, 0x2, 0x2, 0x2, 0x701, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x702, 0x705, - 0x5, 0x177, 0xbc, 0x2, 0x703, 0x706, 0x5, 0x199, 0xcd, 0x2, 0x704, 0x706, - 0x5, 0x183, 0xc2, 0x2, 0x705, 0x703, 0x3, 0x2, 0x2, 0x2, 0x705, 0x704, - 0x3, 0x2, 0x2, 0x2, 0x706, 0x709, 0x3, 0x2, 0x2, 0x2, 0x707, 0x70a, - 0x5, 0x1df, 0xf0, 0x2, 0x708, 0x70a, 0x5, 0x1c5, 0xe3, 0x2, 0x709, 0x707, - 0x3, 0x2, 0x2, 0x2, 0x709, 0x708, 0x3, 0x2, 0x2, 0x2, 0x709, 0x70a, - 0x3, 0x2, 0x2, 0x2, 0x70a, 0x70c, 0x3, 0x2, 0x2, 0x2, 0x70b, 0x70d, - 0x5, 0x1b3, 0xda, 0x2, 0x70c, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x70d, 0x70e, - 0x3, 0x2, 0x2, 0x2, 0x70e, 0x70c, 0x3, 0x2, 0x2, 0x2, 0x70e, 0x70f, - 0x3, 0x2, 0x2, 0x2, 0x70f, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x710, 0x711, - 0x5, 0x175, 0xbb, 0x2, 0x711, 0x715, 0x5, 0x1c7, 0xe4, 0x2, 0x712, 0x714, - 0x5, 0x1b3, 0xda, 0x2, 0x713, 0x712, 0x3, 0x2, 0x2, 0x2, 0x714, 0x717, - 0x3, 0x2, 0x2, 0x2, 0x715, 0x713, 0x3, 0x2, 0x2, 0x2, 0x715, 0x716, - 0x3, 0x2, 0x2, 0x2, 0x716, 0x718, 0x3, 0x2, 0x2, 0x2, 0x717, 0x715, - 0x3, 0x2, 0x2, 0x2, 0x718, 0x71b, 0x5, 0x183, 0xc2, 0x2, 0x719, 0x71c, - 0x5, 0x1df, 0xf0, 0x2, 0x71a, 0x71c, 0x5, 0x1c5, 0xe3, 0x2, 0x71b, 0x719, - 0x3, 0x2, 0x2, 0x2, 0x71b, 0x71a, 0x3, 0x2, 0x2, 0x2, 0x71b, 0x71c, - 0x3, 0x2, 0x2, 0x2, 0x71c, 0x71e, 0x3, 0x2, 0x2, 0x2, 0x71d, 0x71f, - 0x5, 0x1b3, 0xda, 0x2, 0x71e, 0x71d, 0x3, 0x2, 0x2, 0x2, 0x71f, 0x720, - 0x3, 0x2, 0x2, 0x2, 0x720, 0x71e, 0x3, 0x2, 0x2, 0x2, 0x720, 0x721, - 0x3, 0x2, 0x2, 0x2, 0x721, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x722, 0x723, - 0x5, 0x1c7, 0xe4, 0x2, 0x723, 0x724, 0x5, 0x175, 0xbb, 0x2, 0x724, 0x727, - 0x5, 0x183, 0xc2, 0x2, 0x725, 0x728, 0x5, 0x1df, 0xf0, 0x2, 0x726, 0x728, - 0x5, 0x1c5, 0xe3, 0x2, 0x727, 0x725, 0x3, 0x2, 0x2, 0x2, 0x727, 0x726, - 0x3, 0x2, 0x2, 0x2, 0x727, 0x728, 0x3, 0x2, 0x2, 0x2, 0x728, 0x72a, - 0x3, 0x2, 0x2, 0x2, 0x729, 0x72b, 0x5, 0x1b3, 0xda, 0x2, 0x72a, 0x729, - 0x3, 0x2, 0x2, 0x2, 0x72b, 0x72c, 0x3, 0x2, 0x2, 0x2, 0x72c, 0x72a, - 0x3, 0x2, 0x2, 0x2, 0x72c, 0x72d, 0x3, 0x2, 0x2, 0x2, 0x72d, 0x73a, - 0x3, 0x2, 0x2, 0x2, 0x72e, 0x72f, 0x5, 0x175, 0xbb, 0x2, 0x72f, 0x732, - 0x5, 0x183, 0xc2, 0x2, 0x730, 0x733, 0x5, 0x1df, 0xf0, 0x2, 0x731, 0x733, - 0x5, 0x1c5, 0xe3, 0x2, 0x732, 0x730, 0x3, 0x2, 0x2, 0x2, 0x732, 0x731, - 0x3, 0x2, 0x2, 0x2, 0x732, 0x733, 0x3, 0x2, 0x2, 0x2, 0x733, 0x735, - 0x3, 0x2, 0x2, 0x2, 0x734, 0x736, 0x5, 0x1b3, 0xda, 0x2, 0x735, 0x734, - 0x3, 0x2, 0x2, 0x2, 0x736, 0x737, 0x3, 0x2, 0x2, 0x2, 0x737, 0x735, - 0x3, 0x2, 0x2, 0x2, 0x737, 0x738, 0x3, 0x2, 0x2, 0x2, 0x738, 0x73a, - 0x3, 0x2, 0x2, 0x2, 0x739, 0x6ed, 0x3, 0x2, 0x2, 0x2, 0x739, 0x702, - 0x3, 0x2, 0x2, 0x2, 0x739, 0x710, 0x3, 0x2, 0x2, 0x2, 0x739, 0x722, - 0x3, 0x2, 0x2, 0x2, 0x739, 0x72e, 0x3, 0x2, 0x2, 0x2, 0x73a, 0x172, - 0x3, 0x2, 0x2, 0x2, 0x73b, 0x73d, 0x7, 0x32, 0x2, 0x2, 0x73c, 0x73e, - 0x5, 0x1b1, 0xd9, 0x2, 0x73d, 0x73c, 0x3, 0x2, 0x2, 0x2, 0x73e, 0x73f, - 0x3, 0x2, 0x2, 0x2, 0x73f, 0x73d, 0x3, 0x2, 0x2, 0x2, 0x73f, 0x740, - 0x3, 0x2, 0x2, 0x2, 0x740, 0x174, 0x3, 0x2, 0x2, 0x2, 0x741, 0x743, - 0x5, 0x1b3, 0xda, 0x2, 0x742, 0x741, 0x3, 0x2, 0x2, 0x2, 0x743, 0x744, - 0x3, 0x2, 0x2, 0x2, 0x744, 0x742, 0x3, 0x2, 0x2, 0x2, 0x744, 0x745, - 0x3, 0x2, 0x2, 0x2, 0x745, 0x176, 0x3, 0x2, 0x2, 0x2, 0x746, 0x747, - 0x7, 0x32, 0x2, 0x2, 0x747, 0x749, 0x5, 0x1a9, 0xd5, 0x2, 0x748, 0x74a, - 0x5, 0x1b5, 0xdb, 0x2, 0x749, 0x748, 0x3, 0x2, 0x2, 0x2, 0x74a, 0x74b, - 0x3, 0x2, 0x2, 0x2, 0x74b, 0x749, 0x3, 0x2, 0x2, 0x2, 0x74b, 0x74c, - 0x3, 0x2, 0x2, 0x2, 0x74c, 0x178, 0x3, 0x2, 0x2, 0x2, 0x74d, 0x757, - 0x5, 0x1e5, 0xf3, 0x2, 0x74e, 0x756, 0xa, 0x4, 0x2, 0x2, 0x74f, 0x750, - 0x5, 0x1bd, 0xdf, 0x2, 0x750, 0x751, 0xb, 0x2, 0x2, 0x2, 0x751, 0x756, - 0x3, 0x2, 0x2, 0x2, 0x752, 0x753, 0x5, 0x1e5, 0xf3, 0x2, 0x753, 0x754, - 0x5, 0x1e5, 0xf3, 0x2, 0x754, 0x756, 0x3, 0x2, 0x2, 0x2, 0x755, 0x74e, - 0x3, 0x2, 0x2, 0x2, 0x755, 0x74f, 0x3, 0x2, 0x2, 0x2, 0x755, 0x752, - 0x3, 0x2, 0x2, 0x2, 0x756, 0x759, 0x3, 0x2, 0x2, 0x2, 0x757, 0x755, - 0x3, 0x2, 0x2, 0x2, 0x757, 0x758, 0x3, 0x2, 0x2, 0x2, 0x758, 0x75a, - 0x3, 0x2, 0x2, 0x2, 0x759, 0x757, 0x3, 0x2, 0x2, 0x2, 0x75a, 0x75b, - 0x5, 0x1e5, 0xf3, 0x2, 0x75b, 0x17a, 0x3, 0x2, 0x2, 0x2, 0x75c, 0x75d, - 0x9, 0x5, 0x2, 0x2, 0x75d, 0x17c, 0x3, 0x2, 0x2, 0x2, 0x75e, 0x75f, - 0x9, 0x6, 0x2, 0x2, 0x75f, 0x17e, 0x3, 0x2, 0x2, 0x2, 0x760, 0x761, - 0x9, 0x7, 0x2, 0x2, 0x761, 0x180, 0x3, 0x2, 0x2, 0x2, 0x762, 0x763, - 0x9, 0x8, 0x2, 0x2, 0x763, 0x182, 0x3, 0x2, 0x2, 0x2, 0x764, 0x765, - 0x9, 0x9, 0x2, 0x2, 0x765, 0x184, 0x3, 0x2, 0x2, 0x2, 0x766, 0x767, - 0x9, 0xa, 0x2, 0x2, 0x767, 0x186, 0x3, 0x2, 0x2, 0x2, 0x768, 0x769, - 0x9, 0xb, 0x2, 0x2, 0x769, 0x188, 0x3, 0x2, 0x2, 0x2, 0x76a, 0x76b, - 0x9, 0xc, 0x2, 0x2, 0x76b, 0x18a, 0x3, 0x2, 0x2, 0x2, 0x76c, 0x76d, - 0x9, 0xd, 0x2, 0x2, 0x76d, 0x18c, 0x3, 0x2, 0x2, 0x2, 0x76e, 0x76f, - 0x9, 0xe, 0x2, 0x2, 0x76f, 0x18e, 0x3, 0x2, 0x2, 0x2, 0x770, 0x771, - 0x9, 0xf, 0x2, 0x2, 0x771, 0x190, 0x3, 0x2, 0x2, 0x2, 0x772, 0x773, - 0x9, 0x10, 0x2, 0x2, 0x773, 0x192, 0x3, 0x2, 0x2, 0x2, 0x774, 0x775, - 0x9, 0x11, 0x2, 0x2, 0x775, 0x194, 0x3, 0x2, 0x2, 0x2, 0x776, 0x777, - 0x9, 0x12, 0x2, 0x2, 0x777, 0x196, 0x3, 0x2, 0x2, 0x2, 0x778, 0x779, - 0x9, 0x13, 0x2, 0x2, 0x779, 0x198, 0x3, 0x2, 0x2, 0x2, 0x77a, 0x77b, - 0x9, 0x14, 0x2, 0x2, 0x77b, 0x19a, 0x3, 0x2, 0x2, 0x2, 0x77c, 0x77d, - 0x9, 0x15, 0x2, 0x2, 0x77d, 0x19c, 0x3, 0x2, 0x2, 0x2, 0x77e, 0x77f, - 0x9, 0x16, 0x2, 0x2, 0x77f, 0x19e, 0x3, 0x2, 0x2, 0x2, 0x780, 0x781, - 0x9, 0x17, 0x2, 0x2, 0x781, 0x1a0, 0x3, 0x2, 0x2, 0x2, 0x782, 0x783, - 0x9, 0x18, 0x2, 0x2, 0x783, 0x1a2, 0x3, 0x2, 0x2, 0x2, 0x784, 0x785, - 0x9, 0x19, 0x2, 0x2, 0x785, 0x1a4, 0x3, 0x2, 0x2, 0x2, 0x786, 0x787, - 0x9, 0x1a, 0x2, 0x2, 0x787, 0x1a6, 0x3, 0x2, 0x2, 0x2, 0x788, 0x789, - 0x9, 0x1b, 0x2, 0x2, 0x789, 0x1a8, 0x3, 0x2, 0x2, 0x2, 0x78a, 0x78b, - 0x9, 0x1c, 0x2, 0x2, 0x78b, 0x1aa, 0x3, 0x2, 0x2, 0x2, 0x78c, 0x78d, - 0x9, 0x1d, 0x2, 0x2, 0x78d, 0x1ac, 0x3, 0x2, 0x2, 0x2, 0x78e, 0x78f, - 0x9, 0x1e, 0x2, 0x2, 0x78f, 0x1ae, 0x3, 0x2, 0x2, 0x2, 0x790, 0x791, - 0x9, 0x1f, 0x2, 0x2, 0x791, 0x1b0, 0x3, 0x2, 0x2, 0x2, 0x792, 0x793, - 0x9, 0x20, 0x2, 0x2, 0x793, 0x1b2, 0x3, 0x2, 0x2, 0x2, 0x794, 0x795, - 0x9, 0x21, 0x2, 0x2, 0x795, 0x1b4, 0x3, 0x2, 0x2, 0x2, 0x796, 0x797, - 0x9, 0x22, 0x2, 0x2, 0x797, 0x1b6, 0x3, 0x2, 0x2, 0x2, 0x798, 0x799, - 0x7, 0x2f, 0x2, 0x2, 0x799, 0x79a, 0x7, 0x40, 0x2, 0x2, 0x79a, 0x1b8, - 0x3, 0x2, 0x2, 0x2, 0x79b, 0x79c, 0x7, 0x2c, 0x2, 0x2, 0x79c, 0x1ba, - 0x3, 0x2, 0x2, 0x2, 0x79d, 0x79e, 0x7, 0x62, 0x2, 0x2, 0x79e, 0x1bc, - 0x3, 0x2, 0x2, 0x2, 0x79f, 0x7a0, 0x7, 0x5e, 0x2, 0x2, 0x7a0, 0x1be, - 0x3, 0x2, 0x2, 0x2, 0x7a1, 0x7a2, 0x7, 0x3c, 0x2, 0x2, 0x7a2, 0x1c0, - 0x3, 0x2, 0x2, 0x2, 0x7a3, 0x7a4, 0x7, 0x2e, 0x2, 0x2, 0x7a4, 0x1c2, - 0x3, 0x2, 0x2, 0x2, 0x7a5, 0x7a6, 0x7, 0x7e, 0x2, 0x2, 0x7a6, 0x7a7, - 0x7, 0x7e, 0x2, 0x2, 0x7a7, 0x1c4, 0x3, 0x2, 0x2, 0x2, 0x7a8, 0x7a9, - 0x7, 0x2f, 0x2, 0x2, 0x7a9, 0x1c6, 0x3, 0x2, 0x2, 0x2, 0x7aa, 0x7ab, - 0x7, 0x30, 0x2, 0x2, 0x7ab, 0x1c8, 0x3, 0x2, 0x2, 0x2, 0x7ac, 0x7ad, - 0x7, 0x3f, 0x2, 0x2, 0x7ad, 0x7ae, 0x7, 0x3f, 0x2, 0x2, 0x7ae, 0x1ca, - 0x3, 0x2, 0x2, 0x2, 0x7af, 0x7b0, 0x7, 0x3f, 0x2, 0x2, 0x7b0, 0x1cc, - 0x3, 0x2, 0x2, 0x2, 0x7b1, 0x7b2, 0x7, 0x40, 0x2, 0x2, 0x7b2, 0x7b3, - 0x7, 0x3f, 0x2, 0x2, 0x7b3, 0x1ce, 0x3, 0x2, 0x2, 0x2, 0x7b4, 0x7b5, - 0x7, 0x40, 0x2, 0x2, 0x7b5, 0x1d0, 0x3, 0x2, 0x2, 0x2, 0x7b6, 0x7b7, - 0x7, 0x7d, 0x2, 0x2, 0x7b7, 0x1d2, 0x3, 0x2, 0x2, 0x2, 0x7b8, 0x7b9, - 0x7, 0x5d, 0x2, 0x2, 0x7b9, 0x1d4, 0x3, 0x2, 0x2, 0x2, 0x7ba, 0x7bb, - 0x7, 0x3e, 0x2, 0x2, 0x7bb, 0x7bc, 0x7, 0x3f, 0x2, 0x2, 0x7bc, 0x1d6, - 0x3, 0x2, 0x2, 0x2, 0x7bd, 0x7be, 0x7, 0x2a, 0x2, 0x2, 0x7be, 0x1d8, - 0x3, 0x2, 0x2, 0x2, 0x7bf, 0x7c0, 0x7, 0x3e, 0x2, 0x2, 0x7c0, 0x1da, - 0x3, 0x2, 0x2, 0x2, 0x7c1, 0x7c2, 0x7, 0x23, 0x2, 0x2, 0x7c2, 0x7c6, - 0x7, 0x3f, 0x2, 0x2, 0x7c3, 0x7c4, 0x7, 0x3e, 0x2, 0x2, 0x7c4, 0x7c6, - 0x7, 0x40, 0x2, 0x2, 0x7c5, 0x7c1, 0x3, 0x2, 0x2, 0x2, 0x7c5, 0x7c3, - 0x3, 0x2, 0x2, 0x2, 0x7c6, 0x1dc, 0x3, 0x2, 0x2, 0x2, 0x7c7, 0x7c8, - 0x7, 0x27, 0x2, 0x2, 0x7c8, 0x1de, 0x3, 0x2, 0x2, 0x2, 0x7c9, 0x7ca, - 0x7, 0x2d, 0x2, 0x2, 0x7ca, 0x1e0, 0x3, 0x2, 0x2, 0x2, 0x7cb, 0x7cc, - 0x7, 0x41, 0x2, 0x2, 0x7cc, 0x1e2, 0x3, 0x2, 0x2, 0x2, 0x7cd, 0x7ce, - 0x7, 0x24, 0x2, 0x2, 0x7ce, 0x1e4, 0x3, 0x2, 0x2, 0x2, 0x7cf, 0x7d0, - 0x7, 0x29, 0x2, 0x2, 0x7d0, 0x1e6, 0x3, 0x2, 0x2, 0x2, 0x7d1, 0x7d2, - 0x7, 0x7f, 0x2, 0x2, 0x7d2, 0x1e8, 0x3, 0x2, 0x2, 0x2, 0x7d3, 0x7d4, - 0x7, 0x5f, 0x2, 0x2, 0x7d4, 0x1ea, 0x3, 0x2, 0x2, 0x2, 0x7d5, 0x7d6, - 0x7, 0x2b, 0x2, 0x2, 0x7d6, 0x1ec, 0x3, 0x2, 0x2, 0x2, 0x7d7, 0x7d8, - 0x7, 0x3d, 0x2, 0x2, 0x7d8, 0x1ee, 0x3, 0x2, 0x2, 0x2, 0x7d9, 0x7da, - 0x7, 0x31, 0x2, 0x2, 0x7da, 0x1f0, 0x3, 0x2, 0x2, 0x2, 0x7db, 0x7dc, - 0x7, 0x61, 0x2, 0x2, 0x7dc, 0x1f2, 0x3, 0x2, 0x2, 0x2, 0x7dd, 0x7de, - 0x7, 0x31, 0x2, 0x2, 0x7de, 0x7df, 0x7, 0x2c, 0x2, 0x2, 0x7df, 0x7e3, - 0x3, 0x2, 0x2, 0x2, 0x7e0, 0x7e2, 0xb, 0x2, 0x2, 0x2, 0x7e1, 0x7e0, - 0x3, 0x2, 0x2, 0x2, 0x7e2, 0x7e5, 0x3, 0x2, 0x2, 0x2, 0x7e3, 0x7e4, - 0x3, 0x2, 0x2, 0x2, 0x7e3, 0x7e1, 0x3, 0x2, 0x2, 0x2, 0x7e4, 0x7e6, - 0x3, 0x2, 0x2, 0x2, 0x7e5, 0x7e3, 0x3, 0x2, 0x2, 0x2, 0x7e6, 0x7e7, - 0x7, 0x2c, 0x2, 0x2, 0x7e7, 0x7e8, 0x7, 0x31, 0x2, 0x2, 0x7e8, 0x7e9, - 0x3, 0x2, 0x2, 0x2, 0x7e9, 0x7ea, 0x8, 0xfa, 0x2, 0x2, 0x7ea, 0x1f4, - 0x3, 0x2, 0x2, 0x2, 0x7eb, 0x7ec, 0x7, 0x2f, 0x2, 0x2, 0x7ec, 0x7ed, - 0x7, 0x2f, 0x2, 0x2, 0x7ed, 0x7f1, 0x3, 0x2, 0x2, 0x2, 0x7ee, 0x7f0, - 0xa, 0x23, 0x2, 0x2, 0x7ef, 0x7ee, 0x3, 0x2, 0x2, 0x2, 0x7f0, 0x7f3, - 0x3, 0x2, 0x2, 0x2, 0x7f1, 0x7ef, 0x3, 0x2, 0x2, 0x2, 0x7f1, 0x7f2, - 0x3, 0x2, 0x2, 0x2, 0x7f2, 0x7f5, 0x3, 0x2, 0x2, 0x2, 0x7f3, 0x7f1, - 0x3, 0x2, 0x2, 0x2, 0x7f4, 0x7f6, 0x9, 0x24, 0x2, 0x2, 0x7f5, 0x7f4, - 0x3, 0x2, 0x2, 0x2, 0x7f6, 0x7f7, 0x3, 0x2, 0x2, 0x2, 0x7f7, 0x7f8, - 0x8, 0xfb, 0x2, 0x2, 0x7f8, 0x1f6, 0x3, 0x2, 0x2, 0x2, 0x7f9, 0x7fa, - 0x9, 0x25, 0x2, 0x2, 0x7fa, 0x7fb, 0x3, 0x2, 0x2, 0x2, 0x7fb, 0x7fc, - 0x8, 0xfc, 0x2, 0x2, 0x7fc, 0x1f8, 0x3, 0x2, 0x2, 0x2, 0x26, 0x2, 0x237, - 0x413, 0x6b4, 0x6c3, 0x6c8, 0x6ca, 0x6d5, 0x6d7, 0x6e4, 0x6e6, 0x6eb, - 0x6f2, 0x6f7, 0x6fb, 0x700, 0x705, 0x709, 0x70e, 0x715, 0x71b, 0x720, - 0x727, 0x72c, 0x732, 0x737, 0x739, 0x73f, 0x744, 0x74b, 0x755, 0x757, - 0x7c5, 0x7e3, 0x7f1, 0x7f5, 0x3, 0x8, 0x2, 0x2, + 0x5, 0x1a5, 0xd3, 0x2, 0x4f7, 0x4f8, 0x5, 0x193, 0xca, 0x2, 0x4f8, 0x4f9, + 0x5, 0x193, 0xca, 0x2, 0x4f9, 0x4fa, 0x5, 0x1a1, 0xd1, 0x2, 0x4fa, 0xe6, + 0x3, 0x2, 0x2, 0x2, 0x4fb, 0x4fc, 0x5, 0x199, 0xcd, 0x2, 0x4fc, 0x4fd, + 0x5, 0x187, 0xc4, 0x2, 0x4fd, 0x4fe, 0x5, 0x187, 0xc4, 0x2, 0x4fe, 0x4ff, + 0x5, 0x1a1, 0xd1, 0x2, 0x4ff, 0x500, 0x5, 0x185, 0xc3, 0x2, 0x500, 0x501, + 0x5, 0x1a3, 0xd2, 0x2, 0x501, 0xe8, 0x3, 0x2, 0x2, 0x2, 0x502, 0x503, + 0x5, 0x199, 0xcd, 0x2, 0x503, 0x504, 0x5, 0x197, 0xcc, 0x2, 0x504, 0xea, + 0x3, 0x2, 0x2, 0x2, 0x505, 0x506, 0x5, 0x199, 0xcd, 0x2, 0x506, 0x507, + 0x5, 0x19b, 0xce, 0x2, 0x507, 0x508, 0x5, 0x1a3, 0xd2, 0x2, 0x508, 0x509, + 0x5, 0x18d, 0xc7, 0x2, 0x509, 0x50a, 0x5, 0x195, 0xcb, 0x2, 0x50a, 0x50b, + 0x5, 0x18d, 0xc7, 0x2, 0x50b, 0x50c, 0x5, 0x1af, 0xd8, 0x2, 0x50c, 0x50d, + 0x5, 0x185, 0xc3, 0x2, 0x50d, 0xec, 0x3, 0x2, 0x2, 0x2, 0x50e, 0x50f, + 0x5, 0x199, 0xcd, 0x2, 0x50f, 0x510, 0x5, 0x19f, 0xd0, 0x2, 0x510, 0xee, + 0x3, 0x2, 0x2, 0x2, 0x511, 0x512, 0x5, 0x199, 0xcd, 0x2, 0x512, 0x513, + 0x5, 0x19f, 0xd0, 0x2, 0x513, 0x514, 0x5, 0x183, 0xc2, 0x2, 0x514, 0x515, + 0x5, 0x185, 0xc3, 0x2, 0x515, 0x516, 0x5, 0x19f, 0xd0, 0x2, 0x516, 0xf0, + 0x3, 0x2, 0x2, 0x2, 0x517, 0x518, 0x5, 0x199, 0xcd, 0x2, 0x518, 0x519, + 0x5, 0x1a5, 0xd3, 0x2, 0x519, 0x51a, 0x5, 0x1a3, 0xd2, 0x2, 0x51a, 0x51b, + 0x5, 0x185, 0xc3, 0x2, 0x51b, 0x51c, 0x5, 0x19f, 0xd0, 0x2, 0x51c, 0xf2, + 0x3, 0x2, 0x2, 0x2, 0x51d, 0x51e, 0x5, 0x199, 0xcd, 0x2, 0x51e, 0x51f, + 0x5, 0x1a5, 0xd3, 0x2, 0x51f, 0x520, 0x5, 0x1a3, 0xd2, 0x2, 0x520, 0x521, + 0x5, 0x187, 0xc4, 0x2, 0x521, 0x522, 0x5, 0x18d, 0xc7, 0x2, 0x522, 0x523, + 0x5, 0x193, 0xca, 0x2, 0x523, 0x524, 0x5, 0x185, 0xc3, 0x2, 0x524, 0xf4, + 0x3, 0x2, 0x2, 0x2, 0x525, 0x526, 0x5, 0x19b, 0xce, 0x2, 0x526, 0x527, + 0x5, 0x17d, 0xbf, 0x2, 0x527, 0x528, 0x5, 0x19f, 0xd0, 0x2, 0x528, 0x529, + 0x5, 0x1a3, 0xd2, 0x2, 0x529, 0x52a, 0x5, 0x18d, 0xc7, 0x2, 0x52a, 0x52b, + 0x5, 0x1a3, 0xd2, 0x2, 0x52b, 0x52c, 0x5, 0x18d, 0xc7, 0x2, 0x52c, 0x52d, + 0x5, 0x199, 0xcd, 0x2, 0x52d, 0x52e, 0x5, 0x197, 0xcc, 0x2, 0x52e, 0xf6, + 0x3, 0x2, 0x2, 0x2, 0x52f, 0x530, 0x5, 0x19b, 0xce, 0x2, 0x530, 0x531, + 0x5, 0x199, 0xcd, 0x2, 0x531, 0x532, 0x5, 0x19b, 0xce, 0x2, 0x532, 0x533, + 0x5, 0x1a5, 0xd3, 0x2, 0x533, 0x534, 0x5, 0x193, 0xca, 0x2, 0x534, 0x535, + 0x5, 0x17d, 0xbf, 0x2, 0x535, 0x536, 0x5, 0x1a3, 0xd2, 0x2, 0x536, 0x537, + 0x5, 0x185, 0xc3, 0x2, 0x537, 0xf8, 0x3, 0x2, 0x2, 0x2, 0x538, 0x539, + 0x5, 0x19b, 0xce, 0x2, 0x539, 0x53a, 0x5, 0x19f, 0xd0, 0x2, 0x53a, 0x53b, + 0x5, 0x185, 0xc3, 0x2, 0x53b, 0x53c, 0x5, 0x1a9, 0xd5, 0x2, 0x53c, 0x53d, + 0x5, 0x18b, 0xc6, 0x2, 0x53d, 0x53e, 0x5, 0x185, 0xc3, 0x2, 0x53e, 0x53f, + 0x5, 0x19f, 0xd0, 0x2, 0x53f, 0x540, 0x5, 0x185, 0xc3, 0x2, 0x540, 0xfa, + 0x3, 0x2, 0x2, 0x2, 0x541, 0x542, 0x5, 0x19b, 0xce, 0x2, 0x542, 0x543, + 0x5, 0x19f, 0xd0, 0x2, 0x543, 0x544, 0x5, 0x18d, 0xc7, 0x2, 0x544, 0x545, + 0x5, 0x195, 0xcb, 0x2, 0x545, 0x546, 0x5, 0x17d, 0xbf, 0x2, 0x546, 0x547, + 0x5, 0x19f, 0xd0, 0x2, 0x547, 0x548, 0x5, 0x1ad, 0xd7, 0x2, 0x548, 0xfc, + 0x3, 0x2, 0x2, 0x2, 0x549, 0x54a, 0x5, 0x19b, 0xce, 0x2, 0x54a, 0x54b, + 0x5, 0x19f, 0xd0, 0x2, 0x54b, 0x54c, 0x5, 0x199, 0xcd, 0x2, 0x54c, 0x54d, + 0x5, 0x18f, 0xc8, 0x2, 0x54d, 0x54e, 0x5, 0x185, 0xc3, 0x2, 0x54e, 0x54f, + 0x5, 0x181, 0xc1, 0x2, 0x54f, 0x550, 0x5, 0x1a3, 0xd2, 0x2, 0x550, 0x551, + 0x5, 0x18d, 0xc7, 0x2, 0x551, 0x552, 0x5, 0x199, 0xcd, 0x2, 0x552, 0x553, + 0x5, 0x197, 0xcc, 0x2, 0x553, 0xfe, 0x3, 0x2, 0x2, 0x2, 0x554, 0x555, + 0x5, 0x19d, 0xcf, 0x2, 0x555, 0x556, 0x5, 0x1a5, 0xd3, 0x2, 0x556, 0x557, + 0x5, 0x17d, 0xbf, 0x2, 0x557, 0x558, 0x5, 0x19f, 0xd0, 0x2, 0x558, 0x559, + 0x5, 0x1a3, 0xd2, 0x2, 0x559, 0x55a, 0x5, 0x185, 0xc3, 0x2, 0x55a, 0x55b, + 0x5, 0x19f, 0xd0, 0x2, 0x55b, 0x100, 0x3, 0x2, 0x2, 0x2, 0x55c, 0x55d, + 0x5, 0x19f, 0xd0, 0x2, 0x55d, 0x55e, 0x5, 0x17d, 0xbf, 0x2, 0x55e, 0x55f, + 0x5, 0x197, 0xcc, 0x2, 0x55f, 0x560, 0x5, 0x189, 0xc5, 0x2, 0x560, 0x561, + 0x5, 0x185, 0xc3, 0x2, 0x561, 0x102, 0x3, 0x2, 0x2, 0x2, 0x562, 0x563, + 0x5, 0x19f, 0xd0, 0x2, 0x563, 0x564, 0x5, 0x185, 0xc3, 0x2, 0x564, 0x565, + 0x5, 0x193, 0xca, 0x2, 0x565, 0x566, 0x5, 0x199, 0xcd, 0x2, 0x566, 0x567, + 0x5, 0x17d, 0xbf, 0x2, 0x567, 0x568, 0x5, 0x183, 0xc2, 0x2, 0x568, 0x104, + 0x3, 0x2, 0x2, 0x2, 0x569, 0x56a, 0x5, 0x19f, 0xd0, 0x2, 0x56a, 0x56b, + 0x5, 0x185, 0xc3, 0x2, 0x56b, 0x56c, 0x5, 0x195, 0xcb, 0x2, 0x56c, 0x56d, + 0x5, 0x199, 0xcd, 0x2, 0x56d, 0x56e, 0x5, 0x1a7, 0xd4, 0x2, 0x56e, 0x56f, + 0x5, 0x185, 0xc3, 0x2, 0x56f, 0x106, 0x3, 0x2, 0x2, 0x2, 0x570, 0x571, + 0x5, 0x19f, 0xd0, 0x2, 0x571, 0x572, 0x5, 0x185, 0xc3, 0x2, 0x572, 0x573, + 0x5, 0x197, 0xcc, 0x2, 0x573, 0x574, 0x5, 0x17d, 0xbf, 0x2, 0x574, 0x575, + 0x5, 0x195, 0xcb, 0x2, 0x575, 0x576, 0x5, 0x185, 0xc3, 0x2, 0x576, 0x108, + 0x3, 0x2, 0x2, 0x2, 0x577, 0x578, 0x5, 0x19f, 0xd0, 0x2, 0x578, 0x579, + 0x5, 0x185, 0xc3, 0x2, 0x579, 0x57a, 0x5, 0x19b, 0xce, 0x2, 0x57a, 0x57b, + 0x5, 0x193, 0xca, 0x2, 0x57b, 0x57c, 0x5, 0x17d, 0xbf, 0x2, 0x57c, 0x57d, + 0x5, 0x181, 0xc1, 0x2, 0x57d, 0x57e, 0x5, 0x185, 0xc3, 0x2, 0x57e, 0x10a, + 0x3, 0x2, 0x2, 0x2, 0x57f, 0x580, 0x5, 0x19f, 0xd0, 0x2, 0x580, 0x581, + 0x5, 0x185, 0xc3, 0x2, 0x581, 0x582, 0x5, 0x19b, 0xce, 0x2, 0x582, 0x583, + 0x5, 0x193, 0xca, 0x2, 0x583, 0x584, 0x5, 0x18d, 0xc7, 0x2, 0x584, 0x585, + 0x5, 0x181, 0xc1, 0x2, 0x585, 0x586, 0x5, 0x17d, 0xbf, 0x2, 0x586, 0x10c, + 0x3, 0x2, 0x2, 0x2, 0x587, 0x588, 0x5, 0x19f, 0xd0, 0x2, 0x588, 0x589, + 0x5, 0x185, 0xc3, 0x2, 0x589, 0x58a, 0x5, 0x19b, 0xce, 0x2, 0x58a, 0x58b, + 0x5, 0x193, 0xca, 0x2, 0x58b, 0x58c, 0x5, 0x18d, 0xc7, 0x2, 0x58c, 0x58d, + 0x5, 0x181, 0xc1, 0x2, 0x58d, 0x58e, 0x5, 0x17d, 0xbf, 0x2, 0x58e, 0x58f, + 0x5, 0x1a3, 0xd2, 0x2, 0x58f, 0x590, 0x5, 0x185, 0xc3, 0x2, 0x590, 0x591, + 0x5, 0x183, 0xc2, 0x2, 0x591, 0x10e, 0x3, 0x2, 0x2, 0x2, 0x592, 0x593, + 0x5, 0x19f, 0xd0, 0x2, 0x593, 0x594, 0x5, 0x18d, 0xc7, 0x2, 0x594, 0x595, + 0x5, 0x189, 0xc5, 0x2, 0x595, 0x596, 0x5, 0x18b, 0xc6, 0x2, 0x596, 0x597, + 0x5, 0x1a3, 0xd2, 0x2, 0x597, 0x110, 0x3, 0x2, 0x2, 0x2, 0x598, 0x599, + 0x5, 0x19f, 0xd0, 0x2, 0x599, 0x59a, 0x5, 0x199, 0xcd, 0x2, 0x59a, 0x59b, + 0x5, 0x193, 0xca, 0x2, 0x59b, 0x59c, 0x5, 0x193, 0xca, 0x2, 0x59c, 0x59d, + 0x5, 0x1a5, 0xd3, 0x2, 0x59d, 0x59e, 0x5, 0x19b, 0xce, 0x2, 0x59e, 0x112, + 0x3, 0x2, 0x2, 0x2, 0x59f, 0x5a0, 0x5, 0x1a1, 0xd1, 0x2, 0x5a0, 0x5a1, + 0x5, 0x17d, 0xbf, 0x2, 0x5a1, 0x5a2, 0x5, 0x195, 0xcb, 0x2, 0x5a2, 0x5a3, + 0x5, 0x19b, 0xce, 0x2, 0x5a3, 0x5a4, 0x5, 0x193, 0xca, 0x2, 0x5a4, 0x5a5, + 0x5, 0x185, 0xc3, 0x2, 0x5a5, 0x114, 0x3, 0x2, 0x2, 0x2, 0x5a6, 0x5a7, + 0x5, 0x1a1, 0xd1, 0x2, 0x5a7, 0x5a8, 0x5, 0x185, 0xc3, 0x2, 0x5a8, 0x5a9, + 0x5, 0x181, 0xc1, 0x2, 0x5a9, 0x5aa, 0x5, 0x199, 0xcd, 0x2, 0x5aa, 0x5ab, + 0x5, 0x197, 0xcc, 0x2, 0x5ab, 0x5ac, 0x5, 0x183, 0xc2, 0x2, 0x5ac, 0x116, + 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x5ae, 0x5, 0x1a1, 0xd1, 0x2, 0x5ae, 0x5af, + 0x5, 0x185, 0xc3, 0x2, 0x5af, 0x5b0, 0x5, 0x193, 0xca, 0x2, 0x5b0, 0x5b1, + 0x5, 0x185, 0xc3, 0x2, 0x5b1, 0x5b2, 0x5, 0x181, 0xc1, 0x2, 0x5b2, 0x5b3, + 0x5, 0x1a3, 0xd2, 0x2, 0x5b3, 0x118, 0x3, 0x2, 0x2, 0x2, 0x5b4, 0x5b5, + 0x5, 0x1a1, 0xd1, 0x2, 0x5b5, 0x5b6, 0x5, 0x185, 0xc3, 0x2, 0x5b6, 0x5b7, + 0x5, 0x195, 0xcb, 0x2, 0x5b7, 0x5b8, 0x5, 0x18d, 0xc7, 0x2, 0x5b8, 0x11a, + 0x3, 0x2, 0x2, 0x2, 0x5b9, 0x5ba, 0x5, 0x1a1, 0xd1, 0x2, 0x5ba, 0x5bb, + 0x5, 0x185, 0xc3, 0x2, 0x5bb, 0x5bc, 0x5, 0x197, 0xcc, 0x2, 0x5bc, 0x5bd, + 0x5, 0x183, 0xc2, 0x2, 0x5bd, 0x5be, 0x5, 0x1a1, 0xd1, 0x2, 0x5be, 0x11c, + 0x3, 0x2, 0x2, 0x2, 0x5bf, 0x5c0, 0x5, 0x1a1, 0xd1, 0x2, 0x5c0, 0x5c1, + 0x5, 0x185, 0xc3, 0x2, 0x5c1, 0x5c2, 0x5, 0x1a3, 0xd2, 0x2, 0x5c2, 0x11e, + 0x3, 0x2, 0x2, 0x2, 0x5c3, 0x5c4, 0x5, 0x1a1, 0xd1, 0x2, 0x5c4, 0x5c5, + 0x5, 0x185, 0xc3, 0x2, 0x5c5, 0x5c6, 0x5, 0x1a3, 0xd2, 0x2, 0x5c6, 0x5c7, + 0x5, 0x1a3, 0xd2, 0x2, 0x5c7, 0x5c8, 0x5, 0x18d, 0xc7, 0x2, 0x5c8, 0x5c9, + 0x5, 0x197, 0xcc, 0x2, 0x5c9, 0x5ca, 0x5, 0x189, 0xc5, 0x2, 0x5ca, 0x5cb, + 0x5, 0x1a1, 0xd1, 0x2, 0x5cb, 0x120, 0x3, 0x2, 0x2, 0x2, 0x5cc, 0x5cd, + 0x5, 0x1a1, 0xd1, 0x2, 0x5cd, 0x5ce, 0x5, 0x18b, 0xc6, 0x2, 0x5ce, 0x5cf, + 0x5, 0x199, 0xcd, 0x2, 0x5cf, 0x5d0, 0x5, 0x1a9, 0xd5, 0x2, 0x5d0, 0x122, + 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5d2, 0x5, 0x1a1, 0xd1, 0x2, 0x5d2, 0x5d3, + 0x5, 0x199, 0xcd, 0x2, 0x5d3, 0x5d4, 0x5, 0x1a5, 0xd3, 0x2, 0x5d4, 0x5d5, + 0x5, 0x19f, 0xd0, 0x2, 0x5d5, 0x5d6, 0x5, 0x181, 0xc1, 0x2, 0x5d6, 0x5d7, + 0x5, 0x185, 0xc3, 0x2, 0x5d7, 0x124, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5d9, + 0x5, 0x1a1, 0xd1, 0x2, 0x5d9, 0x5da, 0x5, 0x1a3, 0xd2, 0x2, 0x5da, 0x5db, + 0x5, 0x17d, 0xbf, 0x2, 0x5db, 0x5dc, 0x5, 0x19f, 0xd0, 0x2, 0x5dc, 0x5dd, + 0x5, 0x1a3, 0xd2, 0x2, 0x5dd, 0x126, 0x3, 0x2, 0x2, 0x2, 0x5de, 0x5df, + 0x5, 0x1a1, 0xd1, 0x2, 0x5df, 0x5e0, 0x5, 0x1a3, 0xd2, 0x2, 0x5e0, 0x5e1, + 0x5, 0x199, 0xcd, 0x2, 0x5e1, 0x5e2, 0x5, 0x19b, 0xce, 0x2, 0x5e2, 0x128, + 0x3, 0x2, 0x2, 0x2, 0x5e3, 0x5e4, 0x5, 0x1a1, 0xd1, 0x2, 0x5e4, 0x5e5, + 0x5, 0x1a5, 0xd3, 0x2, 0x5e5, 0x5e6, 0x5, 0x17f, 0xc0, 0x2, 0x5e6, 0x5e7, + 0x5, 0x1a1, 0xd1, 0x2, 0x5e7, 0x5e8, 0x5, 0x1a3, 0xd2, 0x2, 0x5e8, 0x5e9, + 0x5, 0x19f, 0xd0, 0x2, 0x5e9, 0x5ea, 0x5, 0x18d, 0xc7, 0x2, 0x5ea, 0x5eb, + 0x5, 0x197, 0xcc, 0x2, 0x5eb, 0x5ec, 0x5, 0x189, 0xc5, 0x2, 0x5ec, 0x12a, + 0x3, 0x2, 0x2, 0x2, 0x5ed, 0x5ee, 0x5, 0x1a1, 0xd1, 0x2, 0x5ee, 0x5ef, + 0x5, 0x1ad, 0xd7, 0x2, 0x5ef, 0x5f0, 0x5, 0x197, 0xcc, 0x2, 0x5f0, 0x5f1, + 0x5, 0x181, 0xc1, 0x2, 0x5f1, 0x12c, 0x3, 0x2, 0x2, 0x2, 0x5f2, 0x5f3, + 0x5, 0x1a1, 0xd1, 0x2, 0x5f3, 0x5f4, 0x5, 0x1ad, 0xd7, 0x2, 0x5f4, 0x5f5, + 0x5, 0x197, 0xcc, 0x2, 0x5f5, 0x5f6, 0x5, 0x1a3, 0xd2, 0x2, 0x5f6, 0x5f7, + 0x5, 0x17d, 0xbf, 0x2, 0x5f7, 0x5f8, 0x5, 0x1ab, 0xd6, 0x2, 0x5f8, 0x12e, + 0x3, 0x2, 0x2, 0x2, 0x5f9, 0x5fa, 0x5, 0x1a1, 0xd1, 0x2, 0x5fa, 0x5fb, + 0x5, 0x1ad, 0xd7, 0x2, 0x5fb, 0x5fc, 0x5, 0x1a1, 0xd1, 0x2, 0x5fc, 0x5fd, + 0x5, 0x1a3, 0xd2, 0x2, 0x5fd, 0x5fe, 0x5, 0x185, 0xc3, 0x2, 0x5fe, 0x5ff, + 0x5, 0x195, 0xcb, 0x2, 0x5ff, 0x130, 0x3, 0x2, 0x2, 0x2, 0x600, 0x601, + 0x5, 0x1a3, 0xd2, 0x2, 0x601, 0x602, 0x5, 0x17d, 0xbf, 0x2, 0x602, 0x603, + 0x5, 0x17f, 0xc0, 0x2, 0x603, 0x604, 0x5, 0x193, 0xca, 0x2, 0x604, 0x605, + 0x5, 0x185, 0xc3, 0x2, 0x605, 0x132, 0x3, 0x2, 0x2, 0x2, 0x606, 0x607, + 0x5, 0x1a3, 0xd2, 0x2, 0x607, 0x608, 0x5, 0x17d, 0xbf, 0x2, 0x608, 0x609, + 0x5, 0x17f, 0xc0, 0x2, 0x609, 0x60a, 0x5, 0x193, 0xca, 0x2, 0x60a, 0x60b, + 0x5, 0x185, 0xc3, 0x2, 0x60b, 0x60c, 0x5, 0x1a1, 0xd1, 0x2, 0x60c, 0x134, + 0x3, 0x2, 0x2, 0x2, 0x60d, 0x60e, 0x5, 0x1a3, 0xd2, 0x2, 0x60e, 0x60f, + 0x5, 0x185, 0xc3, 0x2, 0x60f, 0x610, 0x5, 0x195, 0xcb, 0x2, 0x610, 0x611, + 0x5, 0x19b, 0xce, 0x2, 0x611, 0x612, 0x5, 0x199, 0xcd, 0x2, 0x612, 0x613, + 0x5, 0x19f, 0xd0, 0x2, 0x613, 0x614, 0x5, 0x17d, 0xbf, 0x2, 0x614, 0x615, + 0x5, 0x19f, 0xd0, 0x2, 0x615, 0x616, 0x5, 0x1ad, 0xd7, 0x2, 0x616, 0x136, + 0x3, 0x2, 0x2, 0x2, 0x617, 0x618, 0x5, 0x1a3, 0xd2, 0x2, 0x618, 0x619, + 0x5, 0x185, 0xc3, 0x2, 0x619, 0x61a, 0x5, 0x1a1, 0xd1, 0x2, 0x61a, 0x61b, + 0x5, 0x1a3, 0xd2, 0x2, 0x61b, 0x138, 0x3, 0x2, 0x2, 0x2, 0x61c, 0x61d, + 0x5, 0x1a3, 0xd2, 0x2, 0x61d, 0x61e, 0x5, 0x18b, 0xc6, 0x2, 0x61e, 0x61f, + 0x5, 0x185, 0xc3, 0x2, 0x61f, 0x620, 0x5, 0x197, 0xcc, 0x2, 0x620, 0x13a, + 0x3, 0x2, 0x2, 0x2, 0x621, 0x622, 0x5, 0x1a3, 0xd2, 0x2, 0x622, 0x623, + 0x5, 0x18d, 0xc7, 0x2, 0x623, 0x624, 0x5, 0x185, 0xc3, 0x2, 0x624, 0x625, + 0x5, 0x1a1, 0xd1, 0x2, 0x625, 0x13c, 0x3, 0x2, 0x2, 0x2, 0x626, 0x627, + 0x5, 0x1a3, 0xd2, 0x2, 0x627, 0x628, 0x5, 0x18d, 0xc7, 0x2, 0x628, 0x629, + 0x5, 0x195, 0xcb, 0x2, 0x629, 0x62a, 0x5, 0x185, 0xc3, 0x2, 0x62a, 0x62b, + 0x5, 0x199, 0xcd, 0x2, 0x62b, 0x62c, 0x5, 0x1a5, 0xd3, 0x2, 0x62c, 0x62d, + 0x5, 0x1a3, 0xd2, 0x2, 0x62d, 0x13e, 0x3, 0x2, 0x2, 0x2, 0x62e, 0x62f, + 0x5, 0x1a3, 0xd2, 0x2, 0x62f, 0x630, 0x5, 0x18d, 0xc7, 0x2, 0x630, 0x631, + 0x5, 0x195, 0xcb, 0x2, 0x631, 0x632, 0x5, 0x185, 0xc3, 0x2, 0x632, 0x633, + 0x5, 0x1a1, 0xd1, 0x2, 0x633, 0x634, 0x5, 0x1a3, 0xd2, 0x2, 0x634, 0x635, + 0x5, 0x17d, 0xbf, 0x2, 0x635, 0x636, 0x5, 0x195, 0xcb, 0x2, 0x636, 0x637, + 0x5, 0x19b, 0xce, 0x2, 0x637, 0x140, 0x3, 0x2, 0x2, 0x2, 0x638, 0x639, + 0x5, 0x1a3, 0xd2, 0x2, 0x639, 0x63a, 0x5, 0x199, 0xcd, 0x2, 0x63a, 0x142, + 0x3, 0x2, 0x2, 0x2, 0x63b, 0x63c, 0x5, 0x1a3, 0xd2, 0x2, 0x63c, 0x63d, + 0x5, 0x199, 0xcd, 0x2, 0x63d, 0x63e, 0x5, 0x19b, 0xce, 0x2, 0x63e, 0x144, + 0x3, 0x2, 0x2, 0x2, 0x63f, 0x640, 0x5, 0x1a3, 0xd2, 0x2, 0x640, 0x641, + 0x5, 0x199, 0xcd, 0x2, 0x641, 0x642, 0x5, 0x1a3, 0xd2, 0x2, 0x642, 0x643, + 0x5, 0x17d, 0xbf, 0x2, 0x643, 0x644, 0x5, 0x193, 0xca, 0x2, 0x644, 0x645, + 0x5, 0x1a1, 0xd1, 0x2, 0x645, 0x146, 0x3, 0x2, 0x2, 0x2, 0x646, 0x647, + 0x5, 0x1a3, 0xd2, 0x2, 0x647, 0x648, 0x5, 0x19f, 0xd0, 0x2, 0x648, 0x649, + 0x5, 0x17d, 0xbf, 0x2, 0x649, 0x64a, 0x5, 0x18d, 0xc7, 0x2, 0x64a, 0x64b, + 0x5, 0x193, 0xca, 0x2, 0x64b, 0x64c, 0x5, 0x18d, 0xc7, 0x2, 0x64c, 0x64d, + 0x5, 0x197, 0xcc, 0x2, 0x64d, 0x64e, 0x5, 0x189, 0xc5, 0x2, 0x64e, 0x148, + 0x3, 0x2, 0x2, 0x2, 0x64f, 0x650, 0x5, 0x1a3, 0xd2, 0x2, 0x650, 0x651, + 0x5, 0x19f, 0xd0, 0x2, 0x651, 0x652, 0x5, 0x18d, 0xc7, 0x2, 0x652, 0x653, + 0x5, 0x195, 0xcb, 0x2, 0x653, 0x14a, 0x3, 0x2, 0x2, 0x2, 0x654, 0x655, + 0x5, 0x1a3, 0xd2, 0x2, 0x655, 0x656, 0x5, 0x19f, 0xd0, 0x2, 0x656, 0x657, + 0x5, 0x1a5, 0xd3, 0x2, 0x657, 0x658, 0x5, 0x197, 0xcc, 0x2, 0x658, 0x659, + 0x5, 0x181, 0xc1, 0x2, 0x659, 0x65a, 0x5, 0x17d, 0xbf, 0x2, 0x65a, 0x65b, + 0x5, 0x1a3, 0xd2, 0x2, 0x65b, 0x65c, 0x5, 0x185, 0xc3, 0x2, 0x65c, 0x14c, + 0x3, 0x2, 0x2, 0x2, 0x65d, 0x65e, 0x5, 0x1a3, 0xd2, 0x2, 0x65e, 0x65f, + 0x5, 0x1a3, 0xd2, 0x2, 0x65f, 0x660, 0x5, 0x193, 0xca, 0x2, 0x660, 0x14e, + 0x3, 0x2, 0x2, 0x2, 0x661, 0x662, 0x5, 0x1a3, 0xd2, 0x2, 0x662, 0x663, + 0x5, 0x1ad, 0xd7, 0x2, 0x663, 0x664, 0x5, 0x19b, 0xce, 0x2, 0x664, 0x665, + 0x5, 0x185, 0xc3, 0x2, 0x665, 0x150, 0x3, 0x2, 0x2, 0x2, 0x666, 0x667, + 0x5, 0x1a5, 0xd3, 0x2, 0x667, 0x668, 0x5, 0x197, 0xcc, 0x2, 0x668, 0x669, + 0x5, 0x18d, 0xc7, 0x2, 0x669, 0x66a, 0x5, 0x199, 0xcd, 0x2, 0x66a, 0x66b, + 0x5, 0x197, 0xcc, 0x2, 0x66b, 0x152, 0x3, 0x2, 0x2, 0x2, 0x66c, 0x66d, + 0x5, 0x1a5, 0xd3, 0x2, 0x66d, 0x66e, 0x5, 0x19b, 0xce, 0x2, 0x66e, 0x66f, + 0x5, 0x183, 0xc2, 0x2, 0x66f, 0x670, 0x5, 0x17d, 0xbf, 0x2, 0x670, 0x671, + 0x5, 0x1a3, 0xd2, 0x2, 0x671, 0x672, 0x5, 0x185, 0xc3, 0x2, 0x672, 0x154, + 0x3, 0x2, 0x2, 0x2, 0x673, 0x674, 0x5, 0x1a5, 0xd3, 0x2, 0x674, 0x675, + 0x5, 0x1a1, 0xd1, 0x2, 0x675, 0x676, 0x5, 0x185, 0xc3, 0x2, 0x676, 0x156, + 0x3, 0x2, 0x2, 0x2, 0x677, 0x678, 0x5, 0x1a5, 0xd3, 0x2, 0x678, 0x679, + 0x5, 0x1a1, 0xd1, 0x2, 0x679, 0x67a, 0x5, 0x18d, 0xc7, 0x2, 0x67a, 0x67b, + 0x5, 0x197, 0xcc, 0x2, 0x67b, 0x67c, 0x5, 0x189, 0xc5, 0x2, 0x67c, 0x158, + 0x3, 0x2, 0x2, 0x2, 0x67d, 0x67e, 0x5, 0x1a5, 0xd3, 0x2, 0x67e, 0x67f, + 0x5, 0x1a5, 0xd3, 0x2, 0x67f, 0x680, 0x5, 0x18d, 0xc7, 0x2, 0x680, 0x681, + 0x5, 0x183, 0xc2, 0x2, 0x681, 0x15a, 0x3, 0x2, 0x2, 0x2, 0x682, 0x683, + 0x5, 0x1a7, 0xd4, 0x2, 0x683, 0x684, 0x5, 0x17d, 0xbf, 0x2, 0x684, 0x685, + 0x5, 0x193, 0xca, 0x2, 0x685, 0x686, 0x5, 0x1a5, 0xd3, 0x2, 0x686, 0x687, + 0x5, 0x185, 0xc3, 0x2, 0x687, 0x688, 0x5, 0x1a1, 0xd1, 0x2, 0x688, 0x15c, + 0x3, 0x2, 0x2, 0x2, 0x689, 0x68a, 0x5, 0x1a7, 0xd4, 0x2, 0x68a, 0x68b, + 0x5, 0x18d, 0xc7, 0x2, 0x68b, 0x68c, 0x5, 0x185, 0xc3, 0x2, 0x68c, 0x68d, + 0x5, 0x1a9, 0xd5, 0x2, 0x68d, 0x15e, 0x3, 0x2, 0x2, 0x2, 0x68e, 0x68f, + 0x5, 0x1a7, 0xd4, 0x2, 0x68f, 0x690, 0x5, 0x199, 0xcd, 0x2, 0x690, 0x691, + 0x5, 0x193, 0xca, 0x2, 0x691, 0x692, 0x5, 0x1a5, 0xd3, 0x2, 0x692, 0x693, + 0x5, 0x195, 0xcb, 0x2, 0x693, 0x694, 0x5, 0x185, 0xc3, 0x2, 0x694, 0x160, + 0x3, 0x2, 0x2, 0x2, 0x695, 0x696, 0x5, 0x1a9, 0xd5, 0x2, 0x696, 0x697, + 0x5, 0x17d, 0xbf, 0x2, 0x697, 0x698, 0x5, 0x1a3, 0xd2, 0x2, 0x698, 0x699, + 0x5, 0x181, 0xc1, 0x2, 0x699, 0x69a, 0x5, 0x18b, 0xc6, 0x2, 0x69a, 0x162, + 0x3, 0x2, 0x2, 0x2, 0x69b, 0x69c, 0x5, 0x1a9, 0xd5, 0x2, 0x69c, 0x69d, + 0x5, 0x185, 0xc3, 0x2, 0x69d, 0x69e, 0x5, 0x185, 0xc3, 0x2, 0x69e, 0x69f, + 0x5, 0x191, 0xc9, 0x2, 0x69f, 0x164, 0x3, 0x2, 0x2, 0x2, 0x6a0, 0x6a1, + 0x5, 0x1a9, 0xd5, 0x2, 0x6a1, 0x6a2, 0x5, 0x18b, 0xc6, 0x2, 0x6a2, 0x6a3, + 0x5, 0x185, 0xc3, 0x2, 0x6a3, 0x6a4, 0x5, 0x197, 0xcc, 0x2, 0x6a4, 0x166, + 0x3, 0x2, 0x2, 0x2, 0x6a5, 0x6a6, 0x5, 0x1a9, 0xd5, 0x2, 0x6a6, 0x6a7, + 0x5, 0x18b, 0xc6, 0x2, 0x6a7, 0x6a8, 0x5, 0x185, 0xc3, 0x2, 0x6a8, 0x6a9, + 0x5, 0x19f, 0xd0, 0x2, 0x6a9, 0x6aa, 0x5, 0x185, 0xc3, 0x2, 0x6aa, 0x168, + 0x3, 0x2, 0x2, 0x2, 0x6ab, 0x6ac, 0x5, 0x1a9, 0xd5, 0x2, 0x6ac, 0x6ad, + 0x5, 0x18d, 0xc7, 0x2, 0x6ad, 0x6ae, 0x5, 0x1a3, 0xd2, 0x2, 0x6ae, 0x6af, + 0x5, 0x18b, 0xc6, 0x2, 0x6af, 0x16a, 0x3, 0x2, 0x2, 0x2, 0x6b0, 0x6b1, + 0x5, 0x1ad, 0xd7, 0x2, 0x6b1, 0x6b2, 0x5, 0x185, 0xc3, 0x2, 0x6b2, 0x6b3, + 0x5, 0x17d, 0xbf, 0x2, 0x6b3, 0x6b4, 0x5, 0x19f, 0xd0, 0x2, 0x6b4, 0x6bb, + 0x3, 0x2, 0x2, 0x2, 0x6b5, 0x6b6, 0x5, 0x1ad, 0xd7, 0x2, 0x6b6, 0x6b7, + 0x5, 0x1ad, 0xd7, 0x2, 0x6b7, 0x6b8, 0x5, 0x1ad, 0xd7, 0x2, 0x6b8, 0x6b9, + 0x5, 0x1ad, 0xd7, 0x2, 0x6b9, 0x6bb, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6b0, + 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6b5, 0x3, 0x2, 0x2, 0x2, 0x6bb, 0x16c, + 0x3, 0x2, 0x2, 0x2, 0x6bc, 0x6bd, 0x7, 0x68, 0x2, 0x2, 0x6bd, 0x6be, + 0x7, 0x63, 0x2, 0x2, 0x6be, 0x6bf, 0x7, 0x6e, 0x2, 0x2, 0x6bf, 0x6c0, + 0x7, 0x75, 0x2, 0x2, 0x6c0, 0x6c1, 0x7, 0x67, 0x2, 0x2, 0x6c1, 0x16e, + 0x3, 0x2, 0x2, 0x2, 0x6c2, 0x6c3, 0x7, 0x76, 0x2, 0x2, 0x6c3, 0x6c4, + 0x7, 0x74, 0x2, 0x2, 0x6c4, 0x6c5, 0x7, 0x77, 0x2, 0x2, 0x6c5, 0x6c6, + 0x7, 0x67, 0x2, 0x2, 0x6c6, 0x170, 0x3, 0x2, 0x2, 0x2, 0x6c7, 0x6ca, + 0x5, 0x1b1, 0xd9, 0x2, 0x6c8, 0x6ca, 0x5, 0x1f3, 0xfa, 0x2, 0x6c9, 0x6c7, + 0x3, 0x2, 0x2, 0x2, 0x6c9, 0x6c8, 0x3, 0x2, 0x2, 0x2, 0x6ca, 0x6d0, + 0x3, 0x2, 0x2, 0x2, 0x6cb, 0x6cf, 0x5, 0x1b1, 0xd9, 0x2, 0x6cc, 0x6cf, + 0x5, 0x1f3, 0xfa, 0x2, 0x6cd, 0x6cf, 0x5, 0x1b5, 0xdb, 0x2, 0x6ce, 0x6cb, + 0x3, 0x2, 0x2, 0x2, 0x6ce, 0x6cc, 0x3, 0x2, 0x2, 0x2, 0x6ce, 0x6cd, + 0x3, 0x2, 0x2, 0x2, 0x6cf, 0x6d2, 0x3, 0x2, 0x2, 0x2, 0x6d0, 0x6ce, + 0x3, 0x2, 0x2, 0x2, 0x6d0, 0x6d1, 0x3, 0x2, 0x2, 0x2, 0x6d1, 0x6f2, + 0x3, 0x2, 0x2, 0x2, 0x6d2, 0x6d0, 0x3, 0x2, 0x2, 0x2, 0x6d3, 0x6dd, + 0x5, 0x1bd, 0xdf, 0x2, 0x6d4, 0x6dc, 0xa, 0x2, 0x2, 0x2, 0x6d5, 0x6d6, + 0x5, 0x1bf, 0xe0, 0x2, 0x6d6, 0x6d7, 0xb, 0x2, 0x2, 0x2, 0x6d7, 0x6dc, + 0x3, 0x2, 0x2, 0x2, 0x6d8, 0x6d9, 0x5, 0x1bd, 0xdf, 0x2, 0x6d9, 0x6da, + 0x5, 0x1bd, 0xdf, 0x2, 0x6da, 0x6dc, 0x3, 0x2, 0x2, 0x2, 0x6db, 0x6d4, + 0x3, 0x2, 0x2, 0x2, 0x6db, 0x6d5, 0x3, 0x2, 0x2, 0x2, 0x6db, 0x6d8, + 0x3, 0x2, 0x2, 0x2, 0x6dc, 0x6df, 0x3, 0x2, 0x2, 0x2, 0x6dd, 0x6db, + 0x3, 0x2, 0x2, 0x2, 0x6dd, 0x6de, 0x3, 0x2, 0x2, 0x2, 0x6de, 0x6e0, + 0x3, 0x2, 0x2, 0x2, 0x6df, 0x6dd, 0x3, 0x2, 0x2, 0x2, 0x6e0, 0x6e1, + 0x5, 0x1bd, 0xdf, 0x2, 0x6e1, 0x6f2, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6ec, + 0x5, 0x1e5, 0xf3, 0x2, 0x6e3, 0x6eb, 0xa, 0x3, 0x2, 0x2, 0x6e4, 0x6e5, + 0x5, 0x1bf, 0xe0, 0x2, 0x6e5, 0x6e6, 0xb, 0x2, 0x2, 0x2, 0x6e6, 0x6eb, + 0x3, 0x2, 0x2, 0x2, 0x6e7, 0x6e8, 0x5, 0x1e5, 0xf3, 0x2, 0x6e8, 0x6e9, + 0x5, 0x1e5, 0xf3, 0x2, 0x6e9, 0x6eb, 0x3, 0x2, 0x2, 0x2, 0x6ea, 0x6e3, + 0x3, 0x2, 0x2, 0x2, 0x6ea, 0x6e4, 0x3, 0x2, 0x2, 0x2, 0x6ea, 0x6e7, + 0x3, 0x2, 0x2, 0x2, 0x6eb, 0x6ee, 0x3, 0x2, 0x2, 0x2, 0x6ec, 0x6ea, + 0x3, 0x2, 0x2, 0x2, 0x6ec, 0x6ed, 0x3, 0x2, 0x2, 0x2, 0x6ed, 0x6ef, + 0x3, 0x2, 0x2, 0x2, 0x6ee, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6ef, 0x6f0, + 0x5, 0x1e5, 0xf3, 0x2, 0x6f0, 0x6f2, 0x3, 0x2, 0x2, 0x2, 0x6f1, 0x6c9, + 0x3, 0x2, 0x2, 0x2, 0x6f1, 0x6d3, 0x3, 0x2, 0x2, 0x2, 0x6f1, 0x6e2, + 0x3, 0x2, 0x2, 0x2, 0x6f2, 0x172, 0x3, 0x2, 0x2, 0x2, 0x6f3, 0x6f4, + 0x5, 0x179, 0xbd, 0x2, 0x6f4, 0x6f8, 0x5, 0x1c9, 0xe5, 0x2, 0x6f5, 0x6f7, + 0x5, 0x1b7, 0xdc, 0x2, 0x6f6, 0x6f5, 0x3, 0x2, 0x2, 0x2, 0x6f7, 0x6fa, + 0x3, 0x2, 0x2, 0x2, 0x6f8, 0x6f6, 0x3, 0x2, 0x2, 0x2, 0x6f8, 0x6f9, + 0x3, 0x2, 0x2, 0x2, 0x6f9, 0x6fd, 0x3, 0x2, 0x2, 0x2, 0x6fa, 0x6f8, + 0x3, 0x2, 0x2, 0x2, 0x6fb, 0x6fe, 0x5, 0x19b, 0xce, 0x2, 0x6fc, 0x6fe, + 0x5, 0x185, 0xc3, 0x2, 0x6fd, 0x6fb, 0x3, 0x2, 0x2, 0x2, 0x6fd, 0x6fc, + 0x3, 0x2, 0x2, 0x2, 0x6fe, 0x701, 0x3, 0x2, 0x2, 0x2, 0x6ff, 0x702, + 0x5, 0x1e1, 0xf1, 0x2, 0x700, 0x702, 0x5, 0x1c7, 0xe4, 0x2, 0x701, 0x6ff, + 0x3, 0x2, 0x2, 0x2, 0x701, 0x700, 0x3, 0x2, 0x2, 0x2, 0x701, 0x702, + 0x3, 0x2, 0x2, 0x2, 0x702, 0x704, 0x3, 0x2, 0x2, 0x2, 0x703, 0x705, + 0x5, 0x1b5, 0xdb, 0x2, 0x704, 0x703, 0x3, 0x2, 0x2, 0x2, 0x705, 0x706, + 0x3, 0x2, 0x2, 0x2, 0x706, 0x704, 0x3, 0x2, 0x2, 0x2, 0x706, 0x707, + 0x3, 0x2, 0x2, 0x2, 0x707, 0x740, 0x3, 0x2, 0x2, 0x2, 0x708, 0x70b, + 0x5, 0x179, 0xbd, 0x2, 0x709, 0x70c, 0x5, 0x19b, 0xce, 0x2, 0x70a, 0x70c, + 0x5, 0x185, 0xc3, 0x2, 0x70b, 0x709, 0x3, 0x2, 0x2, 0x2, 0x70b, 0x70a, + 0x3, 0x2, 0x2, 0x2, 0x70c, 0x70f, 0x3, 0x2, 0x2, 0x2, 0x70d, 0x710, + 0x5, 0x1e1, 0xf1, 0x2, 0x70e, 0x710, 0x5, 0x1c7, 0xe4, 0x2, 0x70f, 0x70d, + 0x3, 0x2, 0x2, 0x2, 0x70f, 0x70e, 0x3, 0x2, 0x2, 0x2, 0x70f, 0x710, + 0x3, 0x2, 0x2, 0x2, 0x710, 0x712, 0x3, 0x2, 0x2, 0x2, 0x711, 0x713, + 0x5, 0x1b5, 0xdb, 0x2, 0x712, 0x711, 0x3, 0x2, 0x2, 0x2, 0x713, 0x714, + 0x3, 0x2, 0x2, 0x2, 0x714, 0x712, 0x3, 0x2, 0x2, 0x2, 0x714, 0x715, + 0x3, 0x2, 0x2, 0x2, 0x715, 0x740, 0x3, 0x2, 0x2, 0x2, 0x716, 0x717, + 0x5, 0x177, 0xbc, 0x2, 0x717, 0x71b, 0x5, 0x1c9, 0xe5, 0x2, 0x718, 0x71a, + 0x5, 0x1b5, 0xdb, 0x2, 0x719, 0x718, 0x3, 0x2, 0x2, 0x2, 0x71a, 0x71d, + 0x3, 0x2, 0x2, 0x2, 0x71b, 0x719, 0x3, 0x2, 0x2, 0x2, 0x71b, 0x71c, + 0x3, 0x2, 0x2, 0x2, 0x71c, 0x71e, 0x3, 0x2, 0x2, 0x2, 0x71d, 0x71b, + 0x3, 0x2, 0x2, 0x2, 0x71e, 0x721, 0x5, 0x185, 0xc3, 0x2, 0x71f, 0x722, + 0x5, 0x1e1, 0xf1, 0x2, 0x720, 0x722, 0x5, 0x1c7, 0xe4, 0x2, 0x721, 0x71f, + 0x3, 0x2, 0x2, 0x2, 0x721, 0x720, 0x3, 0x2, 0x2, 0x2, 0x721, 0x722, + 0x3, 0x2, 0x2, 0x2, 0x722, 0x724, 0x3, 0x2, 0x2, 0x2, 0x723, 0x725, + 0x5, 0x1b5, 0xdb, 0x2, 0x724, 0x723, 0x3, 0x2, 0x2, 0x2, 0x725, 0x726, + 0x3, 0x2, 0x2, 0x2, 0x726, 0x724, 0x3, 0x2, 0x2, 0x2, 0x726, 0x727, + 0x3, 0x2, 0x2, 0x2, 0x727, 0x740, 0x3, 0x2, 0x2, 0x2, 0x728, 0x729, + 0x5, 0x1c9, 0xe5, 0x2, 0x729, 0x72a, 0x5, 0x177, 0xbc, 0x2, 0x72a, 0x72d, + 0x5, 0x185, 0xc3, 0x2, 0x72b, 0x72e, 0x5, 0x1e1, 0xf1, 0x2, 0x72c, 0x72e, + 0x5, 0x1c7, 0xe4, 0x2, 0x72d, 0x72b, 0x3, 0x2, 0x2, 0x2, 0x72d, 0x72c, + 0x3, 0x2, 0x2, 0x2, 0x72d, 0x72e, 0x3, 0x2, 0x2, 0x2, 0x72e, 0x730, + 0x3, 0x2, 0x2, 0x2, 0x72f, 0x731, 0x5, 0x1b5, 0xdb, 0x2, 0x730, 0x72f, + 0x3, 0x2, 0x2, 0x2, 0x731, 0x732, 0x3, 0x2, 0x2, 0x2, 0x732, 0x730, + 0x3, 0x2, 0x2, 0x2, 0x732, 0x733, 0x3, 0x2, 0x2, 0x2, 0x733, 0x740, + 0x3, 0x2, 0x2, 0x2, 0x734, 0x735, 0x5, 0x177, 0xbc, 0x2, 0x735, 0x738, + 0x5, 0x185, 0xc3, 0x2, 0x736, 0x739, 0x5, 0x1e1, 0xf1, 0x2, 0x737, 0x739, + 0x5, 0x1c7, 0xe4, 0x2, 0x738, 0x736, 0x3, 0x2, 0x2, 0x2, 0x738, 0x737, + 0x3, 0x2, 0x2, 0x2, 0x738, 0x739, 0x3, 0x2, 0x2, 0x2, 0x739, 0x73b, + 0x3, 0x2, 0x2, 0x2, 0x73a, 0x73c, 0x5, 0x1b5, 0xdb, 0x2, 0x73b, 0x73a, + 0x3, 0x2, 0x2, 0x2, 0x73c, 0x73d, 0x3, 0x2, 0x2, 0x2, 0x73d, 0x73b, + 0x3, 0x2, 0x2, 0x2, 0x73d, 0x73e, 0x3, 0x2, 0x2, 0x2, 0x73e, 0x740, + 0x3, 0x2, 0x2, 0x2, 0x73f, 0x6f3, 0x3, 0x2, 0x2, 0x2, 0x73f, 0x708, + 0x3, 0x2, 0x2, 0x2, 0x73f, 0x716, 0x3, 0x2, 0x2, 0x2, 0x73f, 0x728, + 0x3, 0x2, 0x2, 0x2, 0x73f, 0x734, 0x3, 0x2, 0x2, 0x2, 0x740, 0x174, + 0x3, 0x2, 0x2, 0x2, 0x741, 0x743, 0x7, 0x32, 0x2, 0x2, 0x742, 0x744, + 0x5, 0x1b3, 0xda, 0x2, 0x743, 0x742, 0x3, 0x2, 0x2, 0x2, 0x744, 0x745, + 0x3, 0x2, 0x2, 0x2, 0x745, 0x743, 0x3, 0x2, 0x2, 0x2, 0x745, 0x746, + 0x3, 0x2, 0x2, 0x2, 0x746, 0x176, 0x3, 0x2, 0x2, 0x2, 0x747, 0x749, + 0x5, 0x1b5, 0xdb, 0x2, 0x748, 0x747, 0x3, 0x2, 0x2, 0x2, 0x749, 0x74a, + 0x3, 0x2, 0x2, 0x2, 0x74a, 0x748, 0x3, 0x2, 0x2, 0x2, 0x74a, 0x74b, + 0x3, 0x2, 0x2, 0x2, 0x74b, 0x178, 0x3, 0x2, 0x2, 0x2, 0x74c, 0x74d, + 0x7, 0x32, 0x2, 0x2, 0x74d, 0x74f, 0x5, 0x1ab, 0xd6, 0x2, 0x74e, 0x750, + 0x5, 0x1b7, 0xdc, 0x2, 0x74f, 0x74e, 0x3, 0x2, 0x2, 0x2, 0x750, 0x751, + 0x3, 0x2, 0x2, 0x2, 0x751, 0x74f, 0x3, 0x2, 0x2, 0x2, 0x751, 0x752, + 0x3, 0x2, 0x2, 0x2, 0x752, 0x17a, 0x3, 0x2, 0x2, 0x2, 0x753, 0x75d, + 0x5, 0x1e7, 0xf4, 0x2, 0x754, 0x75c, 0xa, 0x4, 0x2, 0x2, 0x755, 0x756, + 0x5, 0x1bf, 0xe0, 0x2, 0x756, 0x757, 0xb, 0x2, 0x2, 0x2, 0x757, 0x75c, + 0x3, 0x2, 0x2, 0x2, 0x758, 0x759, 0x5, 0x1e7, 0xf4, 0x2, 0x759, 0x75a, + 0x5, 0x1e7, 0xf4, 0x2, 0x75a, 0x75c, 0x3, 0x2, 0x2, 0x2, 0x75b, 0x754, + 0x3, 0x2, 0x2, 0x2, 0x75b, 0x755, 0x3, 0x2, 0x2, 0x2, 0x75b, 0x758, + 0x3, 0x2, 0x2, 0x2, 0x75c, 0x75f, 0x3, 0x2, 0x2, 0x2, 0x75d, 0x75b, + 0x3, 0x2, 0x2, 0x2, 0x75d, 0x75e, 0x3, 0x2, 0x2, 0x2, 0x75e, 0x760, + 0x3, 0x2, 0x2, 0x2, 0x75f, 0x75d, 0x3, 0x2, 0x2, 0x2, 0x760, 0x761, + 0x5, 0x1e7, 0xf4, 0x2, 0x761, 0x17c, 0x3, 0x2, 0x2, 0x2, 0x762, 0x763, + 0x9, 0x5, 0x2, 0x2, 0x763, 0x17e, 0x3, 0x2, 0x2, 0x2, 0x764, 0x765, + 0x9, 0x6, 0x2, 0x2, 0x765, 0x180, 0x3, 0x2, 0x2, 0x2, 0x766, 0x767, + 0x9, 0x7, 0x2, 0x2, 0x767, 0x182, 0x3, 0x2, 0x2, 0x2, 0x768, 0x769, + 0x9, 0x8, 0x2, 0x2, 0x769, 0x184, 0x3, 0x2, 0x2, 0x2, 0x76a, 0x76b, + 0x9, 0x9, 0x2, 0x2, 0x76b, 0x186, 0x3, 0x2, 0x2, 0x2, 0x76c, 0x76d, + 0x9, 0xa, 0x2, 0x2, 0x76d, 0x188, 0x3, 0x2, 0x2, 0x2, 0x76e, 0x76f, + 0x9, 0xb, 0x2, 0x2, 0x76f, 0x18a, 0x3, 0x2, 0x2, 0x2, 0x770, 0x771, + 0x9, 0xc, 0x2, 0x2, 0x771, 0x18c, 0x3, 0x2, 0x2, 0x2, 0x772, 0x773, + 0x9, 0xd, 0x2, 0x2, 0x773, 0x18e, 0x3, 0x2, 0x2, 0x2, 0x774, 0x775, + 0x9, 0xe, 0x2, 0x2, 0x775, 0x190, 0x3, 0x2, 0x2, 0x2, 0x776, 0x777, + 0x9, 0xf, 0x2, 0x2, 0x777, 0x192, 0x3, 0x2, 0x2, 0x2, 0x778, 0x779, + 0x9, 0x10, 0x2, 0x2, 0x779, 0x194, 0x3, 0x2, 0x2, 0x2, 0x77a, 0x77b, + 0x9, 0x11, 0x2, 0x2, 0x77b, 0x196, 0x3, 0x2, 0x2, 0x2, 0x77c, 0x77d, + 0x9, 0x12, 0x2, 0x2, 0x77d, 0x198, 0x3, 0x2, 0x2, 0x2, 0x77e, 0x77f, + 0x9, 0x13, 0x2, 0x2, 0x77f, 0x19a, 0x3, 0x2, 0x2, 0x2, 0x780, 0x781, + 0x9, 0x14, 0x2, 0x2, 0x781, 0x19c, 0x3, 0x2, 0x2, 0x2, 0x782, 0x783, + 0x9, 0x15, 0x2, 0x2, 0x783, 0x19e, 0x3, 0x2, 0x2, 0x2, 0x784, 0x785, + 0x9, 0x16, 0x2, 0x2, 0x785, 0x1a0, 0x3, 0x2, 0x2, 0x2, 0x786, 0x787, + 0x9, 0x17, 0x2, 0x2, 0x787, 0x1a2, 0x3, 0x2, 0x2, 0x2, 0x788, 0x789, + 0x9, 0x18, 0x2, 0x2, 0x789, 0x1a4, 0x3, 0x2, 0x2, 0x2, 0x78a, 0x78b, + 0x9, 0x19, 0x2, 0x2, 0x78b, 0x1a6, 0x3, 0x2, 0x2, 0x2, 0x78c, 0x78d, + 0x9, 0x1a, 0x2, 0x2, 0x78d, 0x1a8, 0x3, 0x2, 0x2, 0x2, 0x78e, 0x78f, + 0x9, 0x1b, 0x2, 0x2, 0x78f, 0x1aa, 0x3, 0x2, 0x2, 0x2, 0x790, 0x791, + 0x9, 0x1c, 0x2, 0x2, 0x791, 0x1ac, 0x3, 0x2, 0x2, 0x2, 0x792, 0x793, + 0x9, 0x1d, 0x2, 0x2, 0x793, 0x1ae, 0x3, 0x2, 0x2, 0x2, 0x794, 0x795, + 0x9, 0x1e, 0x2, 0x2, 0x795, 0x1b0, 0x3, 0x2, 0x2, 0x2, 0x796, 0x797, + 0x9, 0x1f, 0x2, 0x2, 0x797, 0x1b2, 0x3, 0x2, 0x2, 0x2, 0x798, 0x799, + 0x9, 0x20, 0x2, 0x2, 0x799, 0x1b4, 0x3, 0x2, 0x2, 0x2, 0x79a, 0x79b, + 0x9, 0x21, 0x2, 0x2, 0x79b, 0x1b6, 0x3, 0x2, 0x2, 0x2, 0x79c, 0x79d, + 0x9, 0x22, 0x2, 0x2, 0x79d, 0x1b8, 0x3, 0x2, 0x2, 0x2, 0x79e, 0x79f, + 0x7, 0x2f, 0x2, 0x2, 0x79f, 0x7a0, 0x7, 0x40, 0x2, 0x2, 0x7a0, 0x1ba, + 0x3, 0x2, 0x2, 0x2, 0x7a1, 0x7a2, 0x7, 0x2c, 0x2, 0x2, 0x7a2, 0x1bc, + 0x3, 0x2, 0x2, 0x2, 0x7a3, 0x7a4, 0x7, 0x62, 0x2, 0x2, 0x7a4, 0x1be, + 0x3, 0x2, 0x2, 0x2, 0x7a5, 0x7a6, 0x7, 0x5e, 0x2, 0x2, 0x7a6, 0x1c0, + 0x3, 0x2, 0x2, 0x2, 0x7a7, 0x7a8, 0x7, 0x3c, 0x2, 0x2, 0x7a8, 0x1c2, + 0x3, 0x2, 0x2, 0x2, 0x7a9, 0x7aa, 0x7, 0x2e, 0x2, 0x2, 0x7aa, 0x1c4, + 0x3, 0x2, 0x2, 0x2, 0x7ab, 0x7ac, 0x7, 0x7e, 0x2, 0x2, 0x7ac, 0x7ad, + 0x7, 0x7e, 0x2, 0x2, 0x7ad, 0x1c6, 0x3, 0x2, 0x2, 0x2, 0x7ae, 0x7af, + 0x7, 0x2f, 0x2, 0x2, 0x7af, 0x1c8, 0x3, 0x2, 0x2, 0x2, 0x7b0, 0x7b1, + 0x7, 0x30, 0x2, 0x2, 0x7b1, 0x1ca, 0x3, 0x2, 0x2, 0x2, 0x7b2, 0x7b3, + 0x7, 0x3f, 0x2, 0x2, 0x7b3, 0x7b4, 0x7, 0x3f, 0x2, 0x2, 0x7b4, 0x1cc, + 0x3, 0x2, 0x2, 0x2, 0x7b5, 0x7b6, 0x7, 0x3f, 0x2, 0x2, 0x7b6, 0x1ce, + 0x3, 0x2, 0x2, 0x2, 0x7b7, 0x7b8, 0x7, 0x40, 0x2, 0x2, 0x7b8, 0x7b9, + 0x7, 0x3f, 0x2, 0x2, 0x7b9, 0x1d0, 0x3, 0x2, 0x2, 0x2, 0x7ba, 0x7bb, + 0x7, 0x40, 0x2, 0x2, 0x7bb, 0x1d2, 0x3, 0x2, 0x2, 0x2, 0x7bc, 0x7bd, + 0x7, 0x7d, 0x2, 0x2, 0x7bd, 0x1d4, 0x3, 0x2, 0x2, 0x2, 0x7be, 0x7bf, + 0x7, 0x5d, 0x2, 0x2, 0x7bf, 0x1d6, 0x3, 0x2, 0x2, 0x2, 0x7c0, 0x7c1, + 0x7, 0x3e, 0x2, 0x2, 0x7c1, 0x7c2, 0x7, 0x3f, 0x2, 0x2, 0x7c2, 0x1d8, + 0x3, 0x2, 0x2, 0x2, 0x7c3, 0x7c4, 0x7, 0x2a, 0x2, 0x2, 0x7c4, 0x1da, + 0x3, 0x2, 0x2, 0x2, 0x7c5, 0x7c6, 0x7, 0x3e, 0x2, 0x2, 0x7c6, 0x1dc, + 0x3, 0x2, 0x2, 0x2, 0x7c7, 0x7c8, 0x7, 0x23, 0x2, 0x2, 0x7c8, 0x7cc, + 0x7, 0x3f, 0x2, 0x2, 0x7c9, 0x7ca, 0x7, 0x3e, 0x2, 0x2, 0x7ca, 0x7cc, + 0x7, 0x40, 0x2, 0x2, 0x7cb, 0x7c7, 0x3, 0x2, 0x2, 0x2, 0x7cb, 0x7c9, + 0x3, 0x2, 0x2, 0x2, 0x7cc, 0x1de, 0x3, 0x2, 0x2, 0x2, 0x7cd, 0x7ce, + 0x7, 0x27, 0x2, 0x2, 0x7ce, 0x1e0, 0x3, 0x2, 0x2, 0x2, 0x7cf, 0x7d0, + 0x7, 0x2d, 0x2, 0x2, 0x7d0, 0x1e2, 0x3, 0x2, 0x2, 0x2, 0x7d1, 0x7d2, + 0x7, 0x41, 0x2, 0x2, 0x7d2, 0x1e4, 0x3, 0x2, 0x2, 0x2, 0x7d3, 0x7d4, + 0x7, 0x24, 0x2, 0x2, 0x7d4, 0x1e6, 0x3, 0x2, 0x2, 0x2, 0x7d5, 0x7d6, + 0x7, 0x29, 0x2, 0x2, 0x7d6, 0x1e8, 0x3, 0x2, 0x2, 0x2, 0x7d7, 0x7d8, + 0x7, 0x7f, 0x2, 0x2, 0x7d8, 0x1ea, 0x3, 0x2, 0x2, 0x2, 0x7d9, 0x7da, + 0x7, 0x5f, 0x2, 0x2, 0x7da, 0x1ec, 0x3, 0x2, 0x2, 0x2, 0x7db, 0x7dc, + 0x7, 0x2b, 0x2, 0x2, 0x7dc, 0x1ee, 0x3, 0x2, 0x2, 0x2, 0x7dd, 0x7de, + 0x7, 0x3d, 0x2, 0x2, 0x7de, 0x1f0, 0x3, 0x2, 0x2, 0x2, 0x7df, 0x7e0, + 0x7, 0x31, 0x2, 0x2, 0x7e0, 0x1f2, 0x3, 0x2, 0x2, 0x2, 0x7e1, 0x7e2, + 0x7, 0x61, 0x2, 0x2, 0x7e2, 0x1f4, 0x3, 0x2, 0x2, 0x2, 0x7e3, 0x7e4, + 0x7, 0x31, 0x2, 0x2, 0x7e4, 0x7e5, 0x7, 0x2c, 0x2, 0x2, 0x7e5, 0x7e9, + 0x3, 0x2, 0x2, 0x2, 0x7e6, 0x7e8, 0xb, 0x2, 0x2, 0x2, 0x7e7, 0x7e6, + 0x3, 0x2, 0x2, 0x2, 0x7e8, 0x7eb, 0x3, 0x2, 0x2, 0x2, 0x7e9, 0x7ea, + 0x3, 0x2, 0x2, 0x2, 0x7e9, 0x7e7, 0x3, 0x2, 0x2, 0x2, 0x7ea, 0x7ec, + 0x3, 0x2, 0x2, 0x2, 0x7eb, 0x7e9, 0x3, 0x2, 0x2, 0x2, 0x7ec, 0x7ed, + 0x7, 0x2c, 0x2, 0x2, 0x7ed, 0x7ee, 0x7, 0x31, 0x2, 0x2, 0x7ee, 0x7ef, + 0x3, 0x2, 0x2, 0x2, 0x7ef, 0x7f0, 0x8, 0xfb, 0x2, 0x2, 0x7f0, 0x1f6, + 0x3, 0x2, 0x2, 0x2, 0x7f1, 0x7f2, 0x7, 0x2f, 0x2, 0x2, 0x7f2, 0x7f3, + 0x7, 0x2f, 0x2, 0x2, 0x7f3, 0x7f7, 0x3, 0x2, 0x2, 0x2, 0x7f4, 0x7f6, + 0xa, 0x23, 0x2, 0x2, 0x7f5, 0x7f4, 0x3, 0x2, 0x2, 0x2, 0x7f6, 0x7f9, + 0x3, 0x2, 0x2, 0x2, 0x7f7, 0x7f5, 0x3, 0x2, 0x2, 0x2, 0x7f7, 0x7f8, + 0x3, 0x2, 0x2, 0x2, 0x7f8, 0x7fb, 0x3, 0x2, 0x2, 0x2, 0x7f9, 0x7f7, + 0x3, 0x2, 0x2, 0x2, 0x7fa, 0x7fc, 0x9, 0x24, 0x2, 0x2, 0x7fb, 0x7fa, + 0x3, 0x2, 0x2, 0x2, 0x7fc, 0x7fd, 0x3, 0x2, 0x2, 0x2, 0x7fd, 0x7fe, + 0x8, 0xfc, 0x2, 0x2, 0x7fe, 0x1f8, 0x3, 0x2, 0x2, 0x2, 0x7ff, 0x800, + 0x9, 0x25, 0x2, 0x2, 0x800, 0x801, 0x3, 0x2, 0x2, 0x2, 0x801, 0x802, + 0x8, 0xfd, 0x2, 0x2, 0x802, 0x1fa, 0x3, 0x2, 0x2, 0x2, 0x26, 0x2, 0x239, + 0x419, 0x6ba, 0x6c9, 0x6ce, 0x6d0, 0x6db, 0x6dd, 0x6ea, 0x6ec, 0x6f1, + 0x6f8, 0x6fd, 0x701, 0x706, 0x70b, 0x70f, 0x714, 0x71b, 0x721, 0x726, + 0x72d, 0x732, 0x738, 0x73d, 0x73f, 0x745, 0x74a, 0x751, 0x75b, 0x75d, + 0x7cb, 0x7e9, 0x7f7, 0x7fb, 0x3, 0x8, 0x2, 0x2, }; atn::ATNDeserializer deserializer; diff --git a/src/Parsers/New/ClickHouseLexer.g4 b/src/Parsers/New/ClickHouseLexer.g4 index 343b73ff86b..8a1debaf412 100644 --- a/src/Parsers/New/ClickHouseLexer.g4 +++ b/src/Parsers/New/ClickHouseLexer.g4 @@ -16,6 +16,7 @@ ARRAY: A R R A Y; AS: A S; ASCENDING: A S C | A S C E N D I N G; ASOF: A S O F; +AST: A S T; ASYNC: A S Y N C; ATTACH: A T T A C H; BETWEEN: B E T W E E N; diff --git a/src/Parsers/New/ClickHouseLexer.h b/src/Parsers/New/ClickHouseLexer.h index 4de0a30ba2c..1cce0ee0bd7 100644 --- a/src/Parsers/New/ClickHouseLexer.h +++ b/src/Parsers/New/ClickHouseLexer.h @@ -14,46 +14,46 @@ class ClickHouseLexer : public antlr4::Lexer { public: enum { ADD = 1, AFTER = 2, ALIAS = 3, ALL = 4, ALTER = 5, AND = 6, ANTI = 7, - ANY = 8, ARRAY = 9, AS = 10, ASCENDING = 11, ASOF = 12, ASYNC = 13, - ATTACH = 14, BETWEEN = 15, BOTH = 16, BY = 17, CASE = 18, CAST = 19, - CHECK = 20, CLEAR = 21, CLUSTER = 22, CODEC = 23, COLLATE = 24, COLUMN = 25, - COMMENT = 26, CONSTRAINT = 27, CREATE = 28, CROSS = 29, CUBE = 30, DATABASE = 31, - DATABASES = 32, DATE = 33, DAY = 34, DEDUPLICATE = 35, DEFAULT = 36, - DELAY = 37, DELETE = 38, DESC = 39, DESCENDING = 40, DESCRIBE = 41, - DETACH = 42, DICTIONARIES = 43, DICTIONARY = 44, DISK = 45, DISTINCT = 46, - DISTRIBUTED = 47, DROP = 48, ELSE = 49, END = 50, ENGINE = 51, EVENTS = 52, - EXISTS = 53, EXPLAIN = 54, EXPRESSION = 55, EXTRACT = 56, FETCHES = 57, - FINAL = 58, FIRST = 59, FLUSH = 60, FOR = 61, FORMAT = 62, FREEZE = 63, - FROM = 64, FULL = 65, FUNCTION = 66, GLOBAL = 67, GRANULARITY = 68, - GROUP = 69, HAVING = 70, HIERARCHICAL = 71, HOUR = 72, ID = 73, IF = 74, - ILIKE = 75, IN = 76, INDEX = 77, INF = 78, INJECTIVE = 79, INNER = 80, - INSERT = 81, INTERVAL = 82, INTO = 83, IS = 84, IS_OBJECT_ID = 85, JOIN = 86, - KEY = 87, KILL = 88, LAST = 89, LAYOUT = 90, LEADING = 91, LEFT = 92, - LIFETIME = 93, LIKE = 94, LIMIT = 95, LIVE = 96, LOCAL = 97, LOGS = 98, - MATERIALIZED = 99, MATERIALIZE = 100, MAX = 101, MERGES = 102, MIN = 103, - MINUTE = 104, MODIFY = 105, MONTH = 106, MOVE = 107, MUTATION = 108, - NAN_SQL = 109, NO = 110, NOT = 111, NULL_SQL = 112, NULLS = 113, OFFSET = 114, - ON = 115, OPTIMIZE = 116, OR = 117, ORDER = 118, OUTER = 119, OUTFILE = 120, - PARTITION = 121, POPULATE = 122, PREWHERE = 123, PRIMARY = 124, PROJECTION = 125, - QUARTER = 126, RANGE = 127, RELOAD = 128, REMOVE = 129, RENAME = 130, - REPLACE = 131, REPLICA = 132, REPLICATED = 133, RIGHT = 134, ROLLUP = 135, - SAMPLE = 136, SECOND = 137, SELECT = 138, SEMI = 139, SENDS = 140, SET = 141, - SETTINGS = 142, SHOW = 143, SOURCE = 144, START = 145, STOP = 146, SUBSTRING = 147, - SYNC = 148, SYNTAX = 149, SYSTEM = 150, TABLE = 151, TABLES = 152, TEMPORARY = 153, - TEST = 154, THEN = 155, TIES = 156, TIMEOUT = 157, TIMESTAMP = 158, - TO = 159, TOP = 160, TOTALS = 161, TRAILING = 162, TRIM = 163, TRUNCATE = 164, - TTL = 165, TYPE = 166, UNION = 167, UPDATE = 168, USE = 169, USING = 170, - UUID = 171, VALUES = 172, VIEW = 173, VOLUME = 174, WATCH = 175, WEEK = 176, - WHEN = 177, WHERE = 178, WITH = 179, YEAR = 180, JSON_FALSE = 181, JSON_TRUE = 182, - IDENTIFIER = 183, FLOATING_LITERAL = 184, OCTAL_LITERAL = 185, DECIMAL_LITERAL = 186, - HEXADECIMAL_LITERAL = 187, STRING_LITERAL = 188, ARROW = 189, ASTERISK = 190, - BACKQUOTE = 191, BACKSLASH = 192, COLON = 193, COMMA = 194, CONCAT = 195, - DASH = 196, DOT = 197, EQ_DOUBLE = 198, EQ_SINGLE = 199, GE = 200, GT = 201, - LBRACE = 202, LBRACKET = 203, LE = 204, LPAREN = 205, LT = 206, NOT_EQ = 207, - PERCENT = 208, PLUS = 209, QUERY = 210, QUOTE_DOUBLE = 211, QUOTE_SINGLE = 212, - RBRACE = 213, RBRACKET = 214, RPAREN = 215, SEMICOLON = 216, SLASH = 217, - UNDERSCORE = 218, MULTI_LINE_COMMENT = 219, SINGLE_LINE_COMMENT = 220, - WHITESPACE = 221 + ANY = 8, ARRAY = 9, AS = 10, ASCENDING = 11, ASOF = 12, AST = 13, ASYNC = 14, + ATTACH = 15, BETWEEN = 16, BOTH = 17, BY = 18, CASE = 19, CAST = 20, + CHECK = 21, CLEAR = 22, CLUSTER = 23, CODEC = 24, COLLATE = 25, COLUMN = 26, + COMMENT = 27, CONSTRAINT = 28, CREATE = 29, CROSS = 30, CUBE = 31, DATABASE = 32, + DATABASES = 33, DATE = 34, DAY = 35, DEDUPLICATE = 36, DEFAULT = 37, + DELAY = 38, DELETE = 39, DESC = 40, DESCENDING = 41, DESCRIBE = 42, + DETACH = 43, DICTIONARIES = 44, DICTIONARY = 45, DISK = 46, DISTINCT = 47, + DISTRIBUTED = 48, DROP = 49, ELSE = 50, END = 51, ENGINE = 52, EVENTS = 53, + EXISTS = 54, EXPLAIN = 55, EXPRESSION = 56, EXTRACT = 57, FETCHES = 58, + FINAL = 59, FIRST = 60, FLUSH = 61, FOR = 62, FORMAT = 63, FREEZE = 64, + FROM = 65, FULL = 66, FUNCTION = 67, GLOBAL = 68, GRANULARITY = 69, + GROUP = 70, HAVING = 71, HIERARCHICAL = 72, HOUR = 73, ID = 74, IF = 75, + ILIKE = 76, IN = 77, INDEX = 78, INF = 79, INJECTIVE = 80, INNER = 81, + INSERT = 82, INTERVAL = 83, INTO = 84, IS = 85, IS_OBJECT_ID = 86, JOIN = 87, + KEY = 88, KILL = 89, LAST = 90, LAYOUT = 91, LEADING = 92, LEFT = 93, + LIFETIME = 94, LIKE = 95, LIMIT = 96, LIVE = 97, LOCAL = 98, LOGS = 99, + MATERIALIZE = 100, MATERIALIZED = 101, MAX = 102, MERGES = 103, MIN = 104, + MINUTE = 105, MODIFY = 106, MONTH = 107, MOVE = 108, MUTATION = 109, + NAN_SQL = 110, NO = 111, NOT = 112, NULL_SQL = 113, NULLS = 114, OFFSET = 115, + ON = 116, OPTIMIZE = 117, OR = 118, ORDER = 119, OUTER = 120, OUTFILE = 121, + PARTITION = 122, POPULATE = 123, PREWHERE = 124, PRIMARY = 125, PROJECTION = 126, + QUARTER = 127, RANGE = 128, RELOAD = 129, REMOVE = 130, RENAME = 131, + REPLACE = 132, REPLICA = 133, REPLICATED = 134, RIGHT = 135, ROLLUP = 136, + SAMPLE = 137, SECOND = 138, SELECT = 139, SEMI = 140, SENDS = 141, SET = 142, + SETTINGS = 143, SHOW = 144, SOURCE = 145, START = 146, STOP = 147, SUBSTRING = 148, + SYNC = 149, SYNTAX = 150, SYSTEM = 151, TABLE = 152, TABLES = 153, TEMPORARY = 154, + TEST = 155, THEN = 156, TIES = 157, TIMEOUT = 158, TIMESTAMP = 159, + TO = 160, TOP = 161, TOTALS = 162, TRAILING = 163, TRIM = 164, TRUNCATE = 165, + TTL = 166, TYPE = 167, UNION = 168, UPDATE = 169, USE = 170, USING = 171, + UUID = 172, VALUES = 173, VIEW = 174, VOLUME = 175, WATCH = 176, WEEK = 177, + WHEN = 178, WHERE = 179, WITH = 180, YEAR = 181, JSON_FALSE = 182, JSON_TRUE = 183, + IDENTIFIER = 184, FLOATING_LITERAL = 185, OCTAL_LITERAL = 186, DECIMAL_LITERAL = 187, + HEXADECIMAL_LITERAL = 188, STRING_LITERAL = 189, ARROW = 190, ASTERISK = 191, + BACKQUOTE = 192, BACKSLASH = 193, COLON = 194, COMMA = 195, CONCAT = 196, + DASH = 197, DOT = 198, EQ_DOUBLE = 199, EQ_SINGLE = 200, GE = 201, GT = 202, + LBRACE = 203, LBRACKET = 204, LE = 205, LPAREN = 206, LT = 207, NOT_EQ = 208, + PERCENT = 209, PLUS = 210, QUERY = 211, QUOTE_DOUBLE = 212, QUOTE_SINGLE = 213, + RBRACE = 214, RBRACKET = 215, RPAREN = 216, SEMICOLON = 217, SLASH = 218, + UNDERSCORE = 219, MULTI_LINE_COMMENT = 220, SINGLE_LINE_COMMENT = 221, + WHITESPACE = 222 }; ClickHouseLexer(antlr4::CharStream *input); diff --git a/src/Parsers/New/ClickHouseParser.cpp b/src/Parsers/New/ClickHouseParser.cpp index d3ba8563a6f..1be89f63a86 100644 --- a/src/Parsers/New/ClickHouseParser.cpp +++ b/src/Parsers/New/ClickHouseParser.cpp @@ -75,7 +75,6 @@ size_t ClickHouseParser::QueryStmtContext::getRuleIndex() const { return ClickHouseParser::RuleQueryStmt; } - antlrcpp::Any ClickHouseParser::QueryStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitQueryStmt(this); @@ -257,7 +256,6 @@ size_t ClickHouseParser::QueryContext::getRuleIndex() const { return ClickHouseParser::RuleQuery; } - antlrcpp::Any ClickHouseParser::QueryContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitQuery(this); @@ -465,7 +463,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableStmtContext::COMMA(size_t i) { ClickHouseParser::AlterTableStmtContext::AlterTableStmtContext(AlterStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableStmt(this); @@ -557,7 +554,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::AlterTableClauseRepl ClickHouseParser::AlterTableClauseReplaceContext::AlterTableClauseReplaceContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseReplaceContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseReplace(this); @@ -584,7 +580,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::AlterTableClauseModifyOrd ClickHouseParser::AlterTableClauseModifyOrderByContext::AlterTableClauseModifyOrderByContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyOrderByContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModifyOrderBy(this); @@ -607,7 +602,6 @@ ClickHouseParser::WhereClauseContext* ClickHouseParser::AlterTableClauseUpdateCo ClickHouseParser::AlterTableClauseUpdateContext::AlterTableClauseUpdateContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseUpdateContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseUpdate(this); @@ -646,7 +640,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseClea ClickHouseParser::AlterTableClauseClearProjectionContext::AlterTableClauseClearProjectionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseClearProjectionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseClearProjection(this); @@ -685,7 +678,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseModifyRemoveContext::EXIST ClickHouseParser::AlterTableClauseModifyRemoveContext::AlterTableClauseModifyRemoveContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyRemoveContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModifyRemove(this); @@ -708,7 +700,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::AlterTableClauseDeleteCon ClickHouseParser::AlterTableClauseDeleteContext::AlterTableClauseDeleteContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDeleteContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDelete(this); @@ -743,7 +734,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseCommentContext::EXISTS() { ClickHouseParser::AlterTableClauseCommentContext::AlterTableClauseCommentContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseCommentContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseComment(this); @@ -774,7 +764,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseDropColumnContext::EXISTS( ClickHouseParser::AlterTableClauseDropColumnContext::AlterTableClauseDropColumnContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDropColumnContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDropColumn(this); @@ -793,7 +782,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseDeta ClickHouseParser::AlterTableClauseDetachContext::AlterTableClauseDetachContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDetachContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDetach(this); @@ -836,7 +824,6 @@ ClickHouseParser::NestedIdentifierContext* ClickHouseParser::AlterTableClauseAdd ClickHouseParser::AlterTableClauseAddIndexContext::AlterTableClauseAddIndexContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseAddIndexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseAddIndex(this); @@ -855,7 +842,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseDrop ClickHouseParser::AlterTableClauseDropPartitionContext::AlterTableClauseDropPartitionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDropPartitionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDropPartition(this); @@ -894,7 +880,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseMate ClickHouseParser::AlterTableClauseMaterializeIndexContext::AlterTableClauseMaterializeIndexContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseMaterializeIndexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseMaterializeIndex(this); @@ -933,7 +918,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseMate ClickHouseParser::AlterTableClauseMaterializeProjectionContext::AlterTableClauseMaterializeProjectionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseMaterializeProjectionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseMaterializeProjection(this); @@ -976,7 +960,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::AlterTableClauseMove ClickHouseParser::AlterTableClauseMovePartitionContext::AlterTableClauseMovePartitionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseMovePartitionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseMovePartition(this); @@ -1015,7 +998,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseRenameContext::EXISTS() { ClickHouseParser::AlterTableClauseRenameContext::AlterTableClauseRenameContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseRenameContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseRename(this); @@ -1034,7 +1016,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseFree ClickHouseParser::AlterTableClauseFreezePartitionContext::AlterTableClauseFreezePartitionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseFreezePartitionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseFreezePartition(this); @@ -1073,7 +1054,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseClea ClickHouseParser::AlterTableClauseClearColumnContext::AlterTableClauseClearColumnContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseClearColumnContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseClearColumn(this); @@ -1104,7 +1084,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseModifyContext::EXISTS() { ClickHouseParser::AlterTableClauseModifyContext::AlterTableClauseModifyContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModify(this); @@ -1143,7 +1122,6 @@ ClickHouseParser::PartitionClauseContext* ClickHouseParser::AlterTableClauseClea ClickHouseParser::AlterTableClauseClearIndexContext::AlterTableClauseClearIndexContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseClearIndexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseClearIndex(this); @@ -1162,7 +1140,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseRemoveTTLContext::TTL() { ClickHouseParser::AlterTableClauseRemoveTTLContext::AlterTableClauseRemoveTTLContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseRemoveTTLContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseRemoveTTL(this); @@ -1197,7 +1174,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseModifyCodecContext::EXISTS ClickHouseParser::AlterTableClauseModifyCodecContext::AlterTableClauseModifyCodecContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyCodecContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModifyCodec(this); @@ -1224,7 +1200,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::AlterTableClauseAtta ClickHouseParser::AlterTableClauseAttachContext::AlterTableClauseAttachContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseAttachContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseAttach(this); @@ -1255,7 +1230,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseDropProjectionContext::EXI ClickHouseParser::AlterTableClauseDropProjectionContext::AlterTableClauseDropProjectionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDropProjectionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDropProjection(this); @@ -1286,7 +1260,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseDropIndexContext::EXISTS() ClickHouseParser::AlterTableClauseDropIndexContext::AlterTableClauseDropIndexContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseDropIndexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseDropIndex(this); @@ -1325,7 +1298,6 @@ tree::TerminalNode* ClickHouseParser::AlterTableClauseModifyCommentContext::EXIS ClickHouseParser::AlterTableClauseModifyCommentContext::AlterTableClauseModifyCommentContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyCommentContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModifyComment(this); @@ -1344,7 +1316,6 @@ ClickHouseParser::TtlClauseContext* ClickHouseParser::AlterTableClauseModifyTTLC ClickHouseParser::AlterTableClauseModifyTTLContext::AlterTableClauseModifyTTLContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseModifyTTLContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseModifyTTL(this); @@ -1387,7 +1358,6 @@ ClickHouseParser::NestedIdentifierContext* ClickHouseParser::AlterTableClauseAdd ClickHouseParser::AlterTableClauseAddProjectionContext::AlterTableClauseAddProjectionContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseAddProjectionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseAddProjection(this); @@ -1430,7 +1400,6 @@ ClickHouseParser::NestedIdentifierContext* ClickHouseParser::AlterTableClauseAdd ClickHouseParser::AlterTableClauseAddColumnContext::AlterTableClauseAddColumnContext(AlterTableClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AlterTableClauseAddColumnContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlterTableClauseAddColumn(this); @@ -2187,7 +2156,6 @@ size_t ClickHouseParser::AssignmentExprListContext::getRuleIndex() const { return ClickHouseParser::RuleAssignmentExprList; } - antlrcpp::Any ClickHouseParser::AssignmentExprListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAssignmentExprList(this); @@ -2253,7 +2221,6 @@ size_t ClickHouseParser::AssignmentExprContext::getRuleIndex() const { return ClickHouseParser::RuleAssignmentExpr; } - antlrcpp::Any ClickHouseParser::AssignmentExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAssignmentExpr(this); @@ -2322,7 +2289,6 @@ size_t ClickHouseParser::TableColumnPropertyTypeContext::getRuleIndex() const { return ClickHouseParser::RuleTableColumnPropertyType; } - antlrcpp::Any ClickHouseParser::TableColumnPropertyTypeContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableColumnPropertyType(this); @@ -2391,7 +2357,6 @@ size_t ClickHouseParser::PartitionClauseContext::getRuleIndex() const { return ClickHouseParser::RulePartitionClause; } - antlrcpp::Any ClickHouseParser::PartitionClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitPartitionClause(this); @@ -2477,7 +2442,6 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::AttachDictionaryStmtCo ClickHouseParser::AttachDictionaryStmtContext::AttachDictionaryStmtContext(AttachStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::AttachDictionaryStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAttachDictionaryStmt(this); @@ -2547,7 +2511,6 @@ size_t ClickHouseParser::CheckStmtContext::getRuleIndex() const { return ClickHouseParser::RuleCheckStmt; } - antlrcpp::Any ClickHouseParser::CheckStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCheckStmt(this); @@ -2661,7 +2624,6 @@ ClickHouseParser::TableSchemaClauseContext* ClickHouseParser::CreateViewStmtCont ClickHouseParser::CreateViewStmtContext::CreateViewStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateViewStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateViewStmt(this); @@ -2716,7 +2678,6 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::CreateDictionaryStmtCo ClickHouseParser::CreateDictionaryStmtContext::CreateDictionaryStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateDictionaryStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateDictionaryStmt(this); @@ -2763,7 +2724,6 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::CreateDatabaseStmtContext ClickHouseParser::CreateDatabaseStmtContext::CreateDatabaseStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateDatabaseStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateDatabaseStmt(this); @@ -2838,7 +2798,6 @@ tree::TerminalNode* ClickHouseParser::CreateLiveViewStmtContext::DECIMAL_LITERAL ClickHouseParser::CreateLiveViewStmtContext::CreateLiveViewStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateLiveViewStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateLiveViewStmt(this); @@ -2909,7 +2868,6 @@ tree::TerminalNode* ClickHouseParser::CreateMaterializedViewStmtContext::POPULAT ClickHouseParser::CreateMaterializedViewStmtContext::CreateMaterializedViewStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateMaterializedViewStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateMaterializedViewStmt(this); @@ -2972,7 +2930,6 @@ ClickHouseParser::SubqueryClauseContext* ClickHouseParser::CreateTableStmtContex ClickHouseParser::CreateTableStmtContext::CreateTableStmtContext(CreateStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::CreateTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCreateTableStmt(this); @@ -3491,7 +3448,6 @@ size_t ClickHouseParser::DictionarySchemaClauseContext::getRuleIndex() const { return ClickHouseParser::RuleDictionarySchemaClause; } - antlrcpp::Any ClickHouseParser::DictionarySchemaClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionarySchemaClause(this); @@ -3613,7 +3569,6 @@ size_t ClickHouseParser::DictionaryAttrDfntContext::getRuleIndex() const { return ClickHouseParser::RuleDictionaryAttrDfnt; } - antlrcpp::Any ClickHouseParser::DictionaryAttrDfntContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionaryAttrDfnt(this); @@ -3769,7 +3724,6 @@ size_t ClickHouseParser::DictionaryEngineClauseContext::getRuleIndex() const { return ClickHouseParser::RuleDictionaryEngineClause; } - antlrcpp::Any ClickHouseParser::DictionaryEngineClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionaryEngineClause(this); @@ -3896,7 +3850,6 @@ size_t ClickHouseParser::DictionaryPrimaryKeyClauseContext::getRuleIndex() const return ClickHouseParser::RuleDictionaryPrimaryKeyClause; } - antlrcpp::Any ClickHouseParser::DictionaryPrimaryKeyClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionaryPrimaryKeyClause(this); @@ -3961,7 +3914,6 @@ size_t ClickHouseParser::DictionaryArgExprContext::getRuleIndex() const { return ClickHouseParser::RuleDictionaryArgExpr; } - antlrcpp::Any ClickHouseParser::DictionaryArgExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionaryArgExpr(this); @@ -3995,6 +3947,7 @@ ClickHouseParser::DictionaryArgExprContext* ClickHouseParser::dictionaryArgExpr( case ClickHouseParser::AS: case ClickHouseParser::ASCENDING: case ClickHouseParser::ASOF: + case ClickHouseParser::AST: case ClickHouseParser::ASYNC: case ClickHouseParser::ATTACH: case ClickHouseParser::BETWEEN: @@ -4080,8 +4033,8 @@ ClickHouseParser::DictionaryArgExprContext* ClickHouseParser::dictionaryArgExpr( case ClickHouseParser::LIVE: case ClickHouseParser::LOCAL: case ClickHouseParser::LOGS: - case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MATERIALIZE: + case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MAX: case ClickHouseParser::MERGES: case ClickHouseParser::MIN: @@ -4250,7 +4203,6 @@ size_t ClickHouseParser::SourceClauseContext::getRuleIndex() const { return ClickHouseParser::RuleSourceClause; } - antlrcpp::Any ClickHouseParser::SourceClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSourceClause(this); @@ -4291,6 +4243,7 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -4340,9 +4293,9 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -4376,8 +4329,8 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -4400,9 +4353,9 @@ ClickHouseParser::SourceClauseContext* ClickHouseParser::sourceClause() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -4518,7 +4471,6 @@ size_t ClickHouseParser::LifetimeClauseContext::getRuleIndex() const { return ClickHouseParser::RuleLifetimeClause; } - antlrcpp::Any ClickHouseParser::LifetimeClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLifetimeClause(this); @@ -4631,7 +4583,6 @@ size_t ClickHouseParser::LayoutClauseContext::getRuleIndex() const { return ClickHouseParser::RuleLayoutClause; } - antlrcpp::Any ClickHouseParser::LayoutClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLayoutClause(this); @@ -4672,6 +4623,7 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -4721,9 +4673,9 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -4757,8 +4709,8 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -4781,9 +4733,9 @@ ClickHouseParser::LayoutClauseContext* ClickHouseParser::layoutClause() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -4899,7 +4851,6 @@ size_t ClickHouseParser::RangeClauseContext::getRuleIndex() const { return ClickHouseParser::RuleRangeClause; } - antlrcpp::Any ClickHouseParser::RangeClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitRangeClause(this); @@ -4990,7 +4941,6 @@ size_t ClickHouseParser::DictionarySettingsClauseContext::getRuleIndex() const { return ClickHouseParser::RuleDictionarySettingsClause; } - antlrcpp::Any ClickHouseParser::DictionarySettingsClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDictionarySettingsClause(this); @@ -5053,7 +5003,6 @@ size_t ClickHouseParser::ClusterClauseContext::getRuleIndex() const { return ClickHouseParser::RuleClusterClause; } - antlrcpp::Any ClickHouseParser::ClusterClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitClusterClause(this); @@ -5088,6 +5037,7 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::clusterClause() { case ClickHouseParser::AS: case ClickHouseParser::ASCENDING: case ClickHouseParser::ASOF: + case ClickHouseParser::AST: case ClickHouseParser::ASYNC: case ClickHouseParser::ATTACH: case ClickHouseParser::BETWEEN: @@ -5173,8 +5123,8 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::clusterClause() { case ClickHouseParser::LIVE: case ClickHouseParser::LOCAL: case ClickHouseParser::LOGS: - case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MATERIALIZE: + case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MAX: case ClickHouseParser::MERGES: case ClickHouseParser::MIN: @@ -5299,7 +5249,6 @@ size_t ClickHouseParser::UuidClauseContext::getRuleIndex() const { return ClickHouseParser::RuleUuidClause; } - antlrcpp::Any ClickHouseParser::UuidClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitUuidClause(this); @@ -5350,7 +5299,6 @@ size_t ClickHouseParser::DestinationClauseContext::getRuleIndex() const { return ClickHouseParser::RuleDestinationClause; } - antlrcpp::Any ClickHouseParser::DestinationClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDestinationClause(this); @@ -5401,7 +5349,6 @@ size_t ClickHouseParser::SubqueryClauseContext::getRuleIndex() const { return ClickHouseParser::RuleSubqueryClause; } - antlrcpp::Any ClickHouseParser::SubqueryClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSubqueryClause(this); @@ -5460,7 +5407,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::SchemaAsTableClauseC ClickHouseParser::SchemaAsTableClauseContext::SchemaAsTableClauseContext(TableSchemaClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::SchemaAsTableClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSchemaAsTableClause(this); @@ -5479,7 +5425,6 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::SchemaAsFunctionCl ClickHouseParser::SchemaAsFunctionClauseContext::SchemaAsFunctionClauseContext(TableSchemaClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::SchemaAsFunctionClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSchemaAsFunctionClause(this); @@ -5514,7 +5459,6 @@ tree::TerminalNode* ClickHouseParser::SchemaDescriptionClauseContext::COMMA(size ClickHouseParser::SchemaDescriptionClauseContext::SchemaDescriptionClauseContext(TableSchemaClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::SchemaDescriptionClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSchemaDescriptionClause(this); @@ -5652,7 +5596,6 @@ size_t ClickHouseParser::EngineClauseContext::getRuleIndex() const { return ClickHouseParser::RuleEngineClause; } - antlrcpp::Any ClickHouseParser::EngineClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitEngineClause(this); @@ -5780,7 +5723,6 @@ size_t ClickHouseParser::PartitionByClauseContext::getRuleIndex() const { return ClickHouseParser::RulePartitionByClause; } - antlrcpp::Any ClickHouseParser::PartitionByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitPartitionByClause(this); @@ -5837,7 +5779,6 @@ size_t ClickHouseParser::PrimaryKeyClauseContext::getRuleIndex() const { return ClickHouseParser::RulePrimaryKeyClause; } - antlrcpp::Any ClickHouseParser::PrimaryKeyClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitPrimaryKeyClause(this); @@ -5894,7 +5835,6 @@ size_t ClickHouseParser::SampleByClauseContext::getRuleIndex() const { return ClickHouseParser::RuleSampleByClause; } - antlrcpp::Any ClickHouseParser::SampleByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSampleByClause(this); @@ -5959,7 +5899,6 @@ size_t ClickHouseParser::TtlClauseContext::getRuleIndex() const { return ClickHouseParser::RuleTtlClause; } - antlrcpp::Any ClickHouseParser::TtlClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTtlClause(this); @@ -6041,7 +5980,6 @@ size_t ClickHouseParser::EngineExprContext::getRuleIndex() const { return ClickHouseParser::RuleEngineExpr; } - antlrcpp::Any ClickHouseParser::EngineExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitEngineExpr(this); @@ -6094,6 +6032,7 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -6143,9 +6082,9 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -6180,8 +6119,8 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -6206,9 +6145,9 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -6269,12 +6208,12 @@ ClickHouseParser::EngineExprContext* ClickHouseParser::engineExpr() { | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { setState(885); columnExprList(); } @@ -6322,7 +6261,6 @@ ClickHouseParser::TableProjectionDfntContext* ClickHouseParser::TableElementExpr ClickHouseParser::TableElementExprProjectionContext::TableElementExprProjectionContext(TableElementExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableElementExprProjectionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableElementExprProjection(this); @@ -6349,7 +6287,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::TableElementExprConstrain ClickHouseParser::TableElementExprConstraintContext::TableElementExprConstraintContext(TableElementExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableElementExprConstraintContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableElementExprConstraint(this); @@ -6364,7 +6301,6 @@ ClickHouseParser::TableColumnDfntContext* ClickHouseParser::TableElementExprColu ClickHouseParser::TableElementExprColumnContext::TableElementExprColumnContext(TableElementExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableElementExprColumnContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableElementExprColumn(this); @@ -6383,7 +6319,6 @@ ClickHouseParser::TableIndexDfntContext* ClickHouseParser::TableElementExprIndex ClickHouseParser::TableElementExprIndexContext::TableElementExprIndexContext(TableElementExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableElementExprIndexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableElementExprIndex(this); @@ -6498,7 +6433,6 @@ size_t ClickHouseParser::TableColumnDfntContext::getRuleIndex() const { return ClickHouseParser::RuleTableColumnDfnt; } - antlrcpp::Any ClickHouseParser::TableColumnDfntContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableColumnDfnt(this); @@ -6652,7 +6586,6 @@ size_t ClickHouseParser::TableColumnPropertyExprContext::getRuleIndex() const { return ClickHouseParser::RuleTableColumnPropertyExpr; } - antlrcpp::Any ClickHouseParser::TableColumnPropertyExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableColumnPropertyExpr(this); @@ -6729,7 +6662,6 @@ size_t ClickHouseParser::TableIndexDfntContext::getRuleIndex() const { return ClickHouseParser::RuleTableIndexDfnt; } - antlrcpp::Any ClickHouseParser::TableIndexDfntContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableIndexDfnt(this); @@ -6788,7 +6720,6 @@ size_t ClickHouseParser::TableProjectionDfntContext::getRuleIndex() const { return ClickHouseParser::RuleTableProjectionDfnt; } - antlrcpp::Any ClickHouseParser::TableProjectionDfntContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableProjectionDfnt(this); @@ -6859,7 +6790,6 @@ size_t ClickHouseParser::CodecExprContext::getRuleIndex() const { return ClickHouseParser::RuleCodecExpr; } - antlrcpp::Any ClickHouseParser::CodecExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCodecExpr(this); @@ -6935,7 +6865,6 @@ size_t ClickHouseParser::CodecArgExprContext::getRuleIndex() const { return ClickHouseParser::RuleCodecArgExpr; } - antlrcpp::Any ClickHouseParser::CodecArgExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitCodecArgExpr(this); @@ -6978,6 +6907,7 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -7027,9 +6957,9 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -7064,8 +6994,8 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -7090,9 +7020,9 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -7153,12 +7083,12 @@ ClickHouseParser::CodecArgExprContext* ClickHouseParser::codecArgExpr() { | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { setState(964); columnExprList(); } @@ -7211,7 +7141,6 @@ size_t ClickHouseParser::TtlExprContext::getRuleIndex() const { return ClickHouseParser::RuleTtlExpr; } - antlrcpp::Any ClickHouseParser::TtlExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTtlExpr(this); @@ -7299,7 +7228,6 @@ size_t ClickHouseParser::DescribeStmtContext::getRuleIndex() const { return ClickHouseParser::RuleDescribeStmt; } - antlrcpp::Any ClickHouseParser::DescribeStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDescribeStmt(this); @@ -7399,7 +7327,6 @@ ClickHouseParser::ClusterClauseContext* ClickHouseParser::DropDatabaseStmtContex ClickHouseParser::DropDatabaseStmtContext::DropDatabaseStmtContext(DropStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::DropDatabaseStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDropDatabaseStmt(this); @@ -7458,7 +7385,6 @@ tree::TerminalNode* ClickHouseParser::DropTableStmtContext::TEMPORARY() { ClickHouseParser::DropTableStmtContext::DropTableStmtContext(DropStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::DropTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDropTableStmt(this); @@ -7657,7 +7583,6 @@ tree::TerminalNode* ClickHouseParser::ExistsTableStmtContext::TEMPORARY() { ClickHouseParser::ExistsTableStmtContext::ExistsTableStmtContext(ExistsStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ExistsTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitExistsTableStmt(this); @@ -7680,7 +7605,6 @@ ClickHouseParser::DatabaseIdentifierContext* ClickHouseParser::ExistsDatabaseStm ClickHouseParser::ExistsDatabaseStmtContext::ExistsDatabaseStmtContext(ExistsStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ExistsDatabaseStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitExistsDatabaseStmt(this); @@ -7770,31 +7694,59 @@ ClickHouseParser::ExplainStmtContext::ExplainStmtContext(ParserRuleContext *pare : ParserRuleContext(parent, invokingState) { } -tree::TerminalNode* ClickHouseParser::ExplainStmtContext::EXPLAIN() { - return getToken(ClickHouseParser::EXPLAIN, 0); -} - -tree::TerminalNode* ClickHouseParser::ExplainStmtContext::SYNTAX() { - return getToken(ClickHouseParser::SYNTAX, 0); -} - -ClickHouseParser::QueryContext* ClickHouseParser::ExplainStmtContext::query() { - return getRuleContext(0); -} - size_t ClickHouseParser::ExplainStmtContext::getRuleIndex() const { return ClickHouseParser::RuleExplainStmt; } +void ClickHouseParser::ExplainStmtContext::copyFrom(ExplainStmtContext *ctx) { + ParserRuleContext::copyFrom(ctx); +} -antlrcpp::Any ClickHouseParser::ExplainStmtContext::accept(tree::ParseTreeVisitor *visitor) { +//----------------- ExplainSyntaxStmtContext ------------------------------------------------------------------ + +tree::TerminalNode* ClickHouseParser::ExplainSyntaxStmtContext::EXPLAIN() { + return getToken(ClickHouseParser::EXPLAIN, 0); +} + +tree::TerminalNode* ClickHouseParser::ExplainSyntaxStmtContext::SYNTAX() { + return getToken(ClickHouseParser::SYNTAX, 0); +} + +ClickHouseParser::QueryContext* ClickHouseParser::ExplainSyntaxStmtContext::query() { + return getRuleContext(0); +} + +ClickHouseParser::ExplainSyntaxStmtContext::ExplainSyntaxStmtContext(ExplainStmtContext *ctx) { copyFrom(ctx); } + +antlrcpp::Any ClickHouseParser::ExplainSyntaxStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) - return parserVisitor->visitExplainStmt(this); + return parserVisitor->visitExplainSyntaxStmt(this); else return visitor->visitChildren(this); } +//----------------- ExplainASTStmtContext ------------------------------------------------------------------ +tree::TerminalNode* ClickHouseParser::ExplainASTStmtContext::EXPLAIN() { + return getToken(ClickHouseParser::EXPLAIN, 0); +} + +tree::TerminalNode* ClickHouseParser::ExplainASTStmtContext::AST() { + return getToken(ClickHouseParser::AST, 0); +} + +ClickHouseParser::QueryContext* ClickHouseParser::ExplainASTStmtContext::query() { + return getRuleContext(0); +} + +ClickHouseParser::ExplainASTStmtContext::ExplainASTStmtContext(ExplainStmtContext *ctx) { copyFrom(ctx); } + +antlrcpp::Any ClickHouseParser::ExplainASTStmtContext::accept(tree::ParseTreeVisitor *visitor) { + if (auto parserVisitor = dynamic_cast(visitor)) + return parserVisitor->visitExplainASTStmt(this); + else + return visitor->visitChildren(this); +} ClickHouseParser::ExplainStmtContext* ClickHouseParser::explainStmt() { ExplainStmtContext *_localctx = _tracker.createInstance(_ctx, getState()); enterRule(_localctx, 86, ClickHouseParser::RuleExplainStmt); @@ -7803,13 +7755,34 @@ ClickHouseParser::ExplainStmtContext* ClickHouseParser::explainStmt() { exitRule(); }); try { - enterOuterAlt(_localctx, 1); - setState(1034); - match(ClickHouseParser::EXPLAIN); - setState(1035); - match(ClickHouseParser::SYNTAX); - setState(1036); - query(); + setState(1040); + _errHandler->sync(this); + switch (getInterpreter()->adaptivePredict(_input, 119, _ctx)) { + case 1: { + _localctx = dynamic_cast(_tracker.createInstance(_localctx)); + enterOuterAlt(_localctx, 1); + setState(1034); + match(ClickHouseParser::EXPLAIN); + setState(1035); + match(ClickHouseParser::AST); + setState(1036); + query(); + break; + } + + case 2: { + _localctx = dynamic_cast(_tracker.createInstance(_localctx)); + enterOuterAlt(_localctx, 2); + setState(1037); + match(ClickHouseParser::EXPLAIN); + setState(1038); + match(ClickHouseParser::SYNTAX); + setState(1039); + query(); + break; + } + + } } catch (RecognitionException &e) { @@ -7864,7 +7837,6 @@ size_t ClickHouseParser::InsertStmtContext::getRuleIndex() const { return ClickHouseParser::RuleInsertStmt; } - antlrcpp::Any ClickHouseParser::InsertStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitInsertStmt(this); @@ -7881,51 +7853,51 @@ ClickHouseParser::InsertStmtContext* ClickHouseParser::insertStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1038); + setState(1042); match(ClickHouseParser::INSERT); - setState(1039); + setState(1043); match(ClickHouseParser::INTO); - setState(1041); + setState(1045); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 119, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 120, _ctx)) { case 1: { - setState(1040); + setState(1044); match(ClickHouseParser::TABLE); break; } } - setState(1046); + setState(1050); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 120, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 121, _ctx)) { case 1: { - setState(1043); + setState(1047); tableIdentifier(); break; } case 2: { - setState(1044); + setState(1048); match(ClickHouseParser::FUNCTION); - setState(1045); + setState(1049); tableFunctionExpr(); break; } } - setState(1049); + setState(1053); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 121, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 122, _ctx)) { case 1: { - setState(1048); + setState(1052); columnsClause(); break; } } - setState(1051); + setState(1055); dataClause(); } @@ -7973,7 +7945,6 @@ size_t ClickHouseParser::ColumnsClauseContext::getRuleIndex() const { return ClickHouseParser::RuleColumnsClause; } - antlrcpp::Any ClickHouseParser::ColumnsClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnsClause(this); @@ -7991,23 +7962,23 @@ ClickHouseParser::ColumnsClauseContext* ClickHouseParser::columnsClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1053); + setState(1057); match(ClickHouseParser::LPAREN); - setState(1054); + setState(1058); nestedIdentifier(); - setState(1059); + setState(1063); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1055); + setState(1059); match(ClickHouseParser::COMMA); - setState(1056); + setState(1060); nestedIdentifier(); - setState(1061); + setState(1065); _errHandler->sync(this); _la = _input->LA(1); } - setState(1062); + setState(1066); match(ClickHouseParser::RPAREN); } @@ -8043,7 +8014,6 @@ tree::TerminalNode* ClickHouseParser::DataClauseValuesContext::VALUES() { ClickHouseParser::DataClauseValuesContext::DataClauseValuesContext(DataClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::DataClauseValuesContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDataClauseValues(this); @@ -8062,7 +8032,6 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::DataClauseFormatContext:: ClickHouseParser::DataClauseFormatContext::DataClauseFormatContext(DataClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::DataClauseFormatContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDataClauseFormat(this); @@ -8085,7 +8054,6 @@ tree::TerminalNode* ClickHouseParser::DataClauseSelectContext::SEMICOLON() { ClickHouseParser::DataClauseSelectContext::DataClauseSelectContext(DataClauseContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::DataClauseSelectContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDataClauseSelect(this); @@ -8101,15 +8069,15 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { exitRule(); }); try { - setState(1073); + setState(1077); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::FORMAT: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1064); + setState(1068); match(ClickHouseParser::FORMAT); - setState(1065); + setState(1069); identifier(); break; } @@ -8117,7 +8085,7 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { case ClickHouseParser::VALUES: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1066); + setState(1070); match(ClickHouseParser::VALUES); break; } @@ -8127,17 +8095,17 @@ ClickHouseParser::DataClauseContext* ClickHouseParser::dataClause() { case ClickHouseParser::LPAREN: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1067); + setState(1071); selectUnionStmt(); - setState(1069); + setState(1073); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::SEMICOLON) { - setState(1068); + setState(1072); match(ClickHouseParser::SEMICOLON); } - setState(1071); + setState(1075); match(ClickHouseParser::EOF); break; } @@ -8203,7 +8171,6 @@ tree::TerminalNode* ClickHouseParser::KillMutationStmtContext::TEST() { ClickHouseParser::KillMutationStmtContext::KillMutationStmtContext(KillStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::KillMutationStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitKillMutationStmt(this); @@ -8221,28 +8188,28 @@ ClickHouseParser::KillStmtContext* ClickHouseParser::killStmt() { try { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1075); + setState(1079); match(ClickHouseParser::KILL); - setState(1076); + setState(1080); match(ClickHouseParser::MUTATION); - setState(1078); + setState(1082); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1077); + setState(1081); clusterClause(); } - setState(1080); + setState(1084); whereClause(); - setState(1082); + setState(1086); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ASYNC || _la == ClickHouseParser::SYNC || _la == ClickHouseParser::TEST) { - setState(1081); + setState(1085); _la = _input->LA(1); if (!(_la == ClickHouseParser::ASYNC || _la == ClickHouseParser::SYNC @@ -8304,7 +8271,6 @@ size_t ClickHouseParser::OptimizeStmtContext::getRuleIndex() const { return ClickHouseParser::RuleOptimizeStmt; } - antlrcpp::Any ClickHouseParser::OptimizeStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitOptimizeStmt(this); @@ -8322,42 +8288,42 @@ ClickHouseParser::OptimizeStmtContext* ClickHouseParser::optimizeStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1084); - match(ClickHouseParser::OPTIMIZE); - setState(1085); - match(ClickHouseParser::TABLE); - setState(1086); - tableIdentifier(); setState(1088); + match(ClickHouseParser::OPTIMIZE); + setState(1089); + match(ClickHouseParser::TABLE); + setState(1090); + tableIdentifier(); + setState(1092); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1087); + setState(1091); clusterClause(); } - setState(1091); + setState(1095); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::PARTITION) { - setState(1090); + setState(1094); partitionClause(); } - setState(1094); + setState(1098); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FINAL) { - setState(1093); + setState(1097); match(ClickHouseParser::FINAL); } - setState(1097); + setState(1101); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::DEDUPLICATE) { - setState(1096); + setState(1100); match(ClickHouseParser::DEDUPLICATE); } @@ -8418,7 +8384,6 @@ size_t ClickHouseParser::RenameStmtContext::getRuleIndex() const { return ClickHouseParser::RuleRenameStmt; } - antlrcpp::Any ClickHouseParser::RenameStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitRenameStmt(this); @@ -8436,38 +8401,38 @@ ClickHouseParser::RenameStmtContext* ClickHouseParser::renameStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1099); - match(ClickHouseParser::RENAME); - setState(1100); - match(ClickHouseParser::TABLE); - setState(1101); - tableIdentifier(); - setState(1102); - match(ClickHouseParser::TO); setState(1103); + match(ClickHouseParser::RENAME); + setState(1104); + match(ClickHouseParser::TABLE); + setState(1105); tableIdentifier(); - setState(1111); + setState(1106); + match(ClickHouseParser::TO); + setState(1107); + tableIdentifier(); + setState(1115); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1104); + setState(1108); match(ClickHouseParser::COMMA); - setState(1105); + setState(1109); tableIdentifier(); - setState(1106); + setState(1110); match(ClickHouseParser::TO); - setState(1107); + setState(1111); tableIdentifier(); - setState(1113); + setState(1117); _errHandler->sync(this); _la = _input->LA(1); } - setState(1115); + setState(1119); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1114); + setState(1118); clusterClause(); } @@ -8520,7 +8485,6 @@ size_t ClickHouseParser::ProjectionSelectStmtContext::getRuleIndex() const { return ClickHouseParser::RuleProjectionSelectStmt; } - antlrcpp::Any ClickHouseParser::ProjectionSelectStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitProjectionSelectStmt(this); @@ -8538,37 +8502,37 @@ ClickHouseParser::ProjectionSelectStmtContext* ClickHouseParser::projectionSelec }); try { enterOuterAlt(_localctx, 1); - setState(1117); + setState(1121); match(ClickHouseParser::LPAREN); - setState(1119); + setState(1123); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1118); + setState(1122); withClause(); } - setState(1121); + setState(1125); match(ClickHouseParser::SELECT); - setState(1122); + setState(1126); columnExprList(); - setState(1124); + setState(1128); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GROUP) { - setState(1123); + setState(1127); groupByClause(); } - setState(1127); + setState(1131); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ORDER) { - setState(1126); + setState(1130); projectionOrderByClause(); } - setState(1129); + setState(1133); match(ClickHouseParser::RPAREN); } @@ -8616,7 +8580,6 @@ size_t ClickHouseParser::SelectUnionStmtContext::getRuleIndex() const { return ClickHouseParser::RuleSelectUnionStmt; } - antlrcpp::Any ClickHouseParser::SelectUnionStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSelectUnionStmt(this); @@ -8634,19 +8597,19 @@ ClickHouseParser::SelectUnionStmtContext* ClickHouseParser::selectUnionStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1131); + setState(1135); selectStmtWithParens(); - setState(1137); + setState(1141); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::UNION) { - setState(1132); + setState(1136); match(ClickHouseParser::UNION); - setState(1133); + setState(1137); match(ClickHouseParser::ALL); - setState(1134); + setState(1138); selectStmtWithParens(); - setState(1139); + setState(1143); _errHandler->sync(this); _la = _input->LA(1); } @@ -8688,7 +8651,6 @@ size_t ClickHouseParser::SelectStmtWithParensContext::getRuleIndex() const { return ClickHouseParser::RuleSelectStmtWithParens; } - antlrcpp::Any ClickHouseParser::SelectStmtWithParensContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSelectStmtWithParens(this); @@ -8704,24 +8666,24 @@ ClickHouseParser::SelectStmtWithParensContext* ClickHouseParser::selectStmtWithP exitRule(); }); try { - setState(1145); + setState(1149); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::SELECT: case ClickHouseParser::WITH: { enterOuterAlt(_localctx, 1); - setState(1140); + setState(1144); selectStmt(); break; } case ClickHouseParser::LPAREN: { enterOuterAlt(_localctx, 2); - setState(1141); + setState(1145); match(ClickHouseParser::LPAREN); - setState(1142); + setState(1146); selectUnionStmt(); - setState(1143); + setState(1147); match(ClickHouseParser::RPAREN); break; } @@ -8831,7 +8793,6 @@ size_t ClickHouseParser::SelectStmtContext::getRuleIndex() const { return ClickHouseParser::RuleSelectStmt; } - antlrcpp::Any ClickHouseParser::SelectStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSelectStmt(this); @@ -8849,90 +8810,90 @@ ClickHouseParser::SelectStmtContext* ClickHouseParser::selectStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1148); + setState(1152); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1147); + setState(1151); withClause(); } - setState(1150); + setState(1154); match(ClickHouseParser::SELECT); - setState(1152); + setState(1156); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 139, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 140, _ctx)) { case 1: { - setState(1151); + setState(1155); match(ClickHouseParser::DISTINCT); break; } } - setState(1155); + setState(1159); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 140, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 141, _ctx)) { case 1: { - setState(1154); + setState(1158); topClause(); break; } } - setState(1157); + setState(1161); columnExprList(); - setState(1159); + setState(1163); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM) { - setState(1158); + setState(1162); fromClause(); } - setState(1162); + setState(1166); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ARRAY || _la == ClickHouseParser::INNER || _la == ClickHouseParser::LEFT) { - setState(1161); + setState(1165); arrayJoinClause(); } - setState(1165); + setState(1169); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::PREWHERE) { - setState(1164); + setState(1168); prewhereClause(); } - setState(1168); + setState(1172); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WHERE) { - setState(1167); + setState(1171); whereClause(); } - setState(1171); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::GROUP) { - setState(1170); - groupByClause(); - } setState(1175); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 146, _ctx)) { - case 1: { - setState(1173); - match(ClickHouseParser::WITH); + _la = _input->LA(1); + if (_la == ClickHouseParser::GROUP) { setState(1174); + groupByClause(); + } + setState(1179); + _errHandler->sync(this); + + switch (getInterpreter()->adaptivePredict(_input, 147, _ctx)) { + case 1: { + setState(1177); + match(ClickHouseParser::WITH); + setState(1178); _la = _input->LA(1); if (!(_la == ClickHouseParser::CUBE || _la == ClickHouseParser::ROLLUP)) { _errHandler->recoverInline(this); @@ -8945,57 +8906,57 @@ ClickHouseParser::SelectStmtContext* ClickHouseParser::selectStmt() { } } - setState(1179); + setState(1183); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1177); + setState(1181); match(ClickHouseParser::WITH); - setState(1178); + setState(1182); match(ClickHouseParser::TOTALS); } - setState(1182); + setState(1186); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::HAVING) { - setState(1181); + setState(1185); havingClause(); } - setState(1185); + setState(1189); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ORDER) { - setState(1184); + setState(1188); orderByClause(); } - setState(1188); + setState(1192); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 150, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 151, _ctx)) { case 1: { - setState(1187); + setState(1191); limitByClause(); break; } } - setState(1191); + setState(1195); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LIMIT) { - setState(1190); + setState(1194); limitClause(); } - setState(1194); + setState(1198); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::SETTINGS) { - setState(1193); + setState(1197); settingsClause(); } @@ -9028,7 +8989,6 @@ size_t ClickHouseParser::WithClauseContext::getRuleIndex() const { return ClickHouseParser::RuleWithClause; } - antlrcpp::Any ClickHouseParser::WithClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitWithClause(this); @@ -9045,9 +9005,9 @@ ClickHouseParser::WithClauseContext* ClickHouseParser::withClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1196); + setState(1200); match(ClickHouseParser::WITH); - setState(1197); + setState(1201); columnExprList(); } @@ -9087,7 +9047,6 @@ size_t ClickHouseParser::TopClauseContext::getRuleIndex() const { return ClickHouseParser::RuleTopClause; } - antlrcpp::Any ClickHouseParser::TopClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTopClause(this); @@ -9104,18 +9063,18 @@ ClickHouseParser::TopClauseContext* ClickHouseParser::topClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1199); - match(ClickHouseParser::TOP); - setState(1200); - match(ClickHouseParser::DECIMAL_LITERAL); setState(1203); + match(ClickHouseParser::TOP); + setState(1204); + match(ClickHouseParser::DECIMAL_LITERAL); + setState(1207); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 153, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 154, _ctx)) { case 1: { - setState(1201); + setState(1205); match(ClickHouseParser::WITH); - setState(1202); + setState(1206); match(ClickHouseParser::TIES); break; } @@ -9151,7 +9110,6 @@ size_t ClickHouseParser::FromClauseContext::getRuleIndex() const { return ClickHouseParser::RuleFromClause; } - antlrcpp::Any ClickHouseParser::FromClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitFromClause(this); @@ -9168,9 +9126,9 @@ ClickHouseParser::FromClauseContext* ClickHouseParser::fromClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1205); + setState(1209); match(ClickHouseParser::FROM); - setState(1206); + setState(1210); joinExpr(0); } @@ -9214,7 +9172,6 @@ size_t ClickHouseParser::ArrayJoinClauseContext::getRuleIndex() const { return ClickHouseParser::RuleArrayJoinClause; } - antlrcpp::Any ClickHouseParser::ArrayJoinClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitArrayJoinClause(this); @@ -9232,14 +9189,14 @@ ClickHouseParser::ArrayJoinClauseContext* ClickHouseParser::arrayJoinClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1209); + setState(1213); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::INNER || _la == ClickHouseParser::LEFT) { - setState(1208); + setState(1212); _la = _input->LA(1); if (!(_la == ClickHouseParser::INNER @@ -9251,11 +9208,11 @@ ClickHouseParser::ArrayJoinClauseContext* ClickHouseParser::arrayJoinClause() { consume(); } } - setState(1211); + setState(1215); match(ClickHouseParser::ARRAY); - setState(1212); + setState(1216); match(ClickHouseParser::JOIN); - setState(1213); + setState(1217); columnExprList(); } @@ -9287,7 +9244,6 @@ size_t ClickHouseParser::PrewhereClauseContext::getRuleIndex() const { return ClickHouseParser::RulePrewhereClause; } - antlrcpp::Any ClickHouseParser::PrewhereClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitPrewhereClause(this); @@ -9304,9 +9260,9 @@ ClickHouseParser::PrewhereClauseContext* ClickHouseParser::prewhereClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1215); + setState(1219); match(ClickHouseParser::PREWHERE); - setState(1216); + setState(1220); columnExpr(0); } @@ -9338,7 +9294,6 @@ size_t ClickHouseParser::WhereClauseContext::getRuleIndex() const { return ClickHouseParser::RuleWhereClause; } - antlrcpp::Any ClickHouseParser::WhereClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitWhereClause(this); @@ -9355,9 +9310,9 @@ ClickHouseParser::WhereClauseContext* ClickHouseParser::whereClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1218); + setState(1222); match(ClickHouseParser::WHERE); - setState(1219); + setState(1223); columnExpr(0); } @@ -9409,7 +9364,6 @@ size_t ClickHouseParser::GroupByClauseContext::getRuleIndex() const { return ClickHouseParser::RuleGroupByClause; } - antlrcpp::Any ClickHouseParser::GroupByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitGroupByClause(this); @@ -9427,15 +9381,15 @@ ClickHouseParser::GroupByClauseContext* ClickHouseParser::groupByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1221); + setState(1225); match(ClickHouseParser::GROUP); - setState(1222); + setState(1226); match(ClickHouseParser::BY); - setState(1229); + setState(1233); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 155, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 156, _ctx)) { case 1: { - setState(1223); + setState(1227); _la = _input->LA(1); if (!(_la == ClickHouseParser::CUBE || _la == ClickHouseParser::ROLLUP)) { _errHandler->recoverInline(this); @@ -9444,17 +9398,17 @@ ClickHouseParser::GroupByClauseContext* ClickHouseParser::groupByClause() { _errHandler->reportMatch(this); consume(); } - setState(1224); + setState(1228); match(ClickHouseParser::LPAREN); - setState(1225); + setState(1229); columnExprList(); - setState(1226); + setState(1230); match(ClickHouseParser::RPAREN); break; } case 2: { - setState(1228); + setState(1232); columnExprList(); break; } @@ -9490,7 +9444,6 @@ size_t ClickHouseParser::HavingClauseContext::getRuleIndex() const { return ClickHouseParser::RuleHavingClause; } - antlrcpp::Any ClickHouseParser::HavingClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitHavingClause(this); @@ -9507,9 +9460,9 @@ ClickHouseParser::HavingClauseContext* ClickHouseParser::havingClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1231); + setState(1235); match(ClickHouseParser::HAVING); - setState(1232); + setState(1236); columnExpr(0); } @@ -9545,7 +9498,6 @@ size_t ClickHouseParser::OrderByClauseContext::getRuleIndex() const { return ClickHouseParser::RuleOrderByClause; } - antlrcpp::Any ClickHouseParser::OrderByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitOrderByClause(this); @@ -9562,11 +9514,11 @@ ClickHouseParser::OrderByClauseContext* ClickHouseParser::orderByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1234); + setState(1238); match(ClickHouseParser::ORDER); - setState(1235); + setState(1239); match(ClickHouseParser::BY); - setState(1236); + setState(1240); orderExprList(); } @@ -9602,7 +9554,6 @@ size_t ClickHouseParser::ProjectionOrderByClauseContext::getRuleIndex() const { return ClickHouseParser::RuleProjectionOrderByClause; } - antlrcpp::Any ClickHouseParser::ProjectionOrderByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitProjectionOrderByClause(this); @@ -9619,11 +9570,11 @@ ClickHouseParser::ProjectionOrderByClauseContext* ClickHouseParser::projectionOr }); try { enterOuterAlt(_localctx, 1); - setState(1238); + setState(1242); match(ClickHouseParser::ORDER); - setState(1239); + setState(1243); match(ClickHouseParser::BY); - setState(1240); + setState(1244); columnExprList(); } @@ -9663,7 +9614,6 @@ size_t ClickHouseParser::LimitByClauseContext::getRuleIndex() const { return ClickHouseParser::RuleLimitByClause; } - antlrcpp::Any ClickHouseParser::LimitByClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLimitByClause(this); @@ -9680,13 +9630,13 @@ ClickHouseParser::LimitByClauseContext* ClickHouseParser::limitByClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1242); + setState(1246); match(ClickHouseParser::LIMIT); - setState(1243); + setState(1247); limitExpr(); - setState(1244); + setState(1248); match(ClickHouseParser::BY); - setState(1245); + setState(1249); columnExprList(); } @@ -9726,7 +9676,6 @@ size_t ClickHouseParser::LimitClauseContext::getRuleIndex() const { return ClickHouseParser::RuleLimitClause; } - antlrcpp::Any ClickHouseParser::LimitClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLimitClause(this); @@ -9744,18 +9693,18 @@ ClickHouseParser::LimitClauseContext* ClickHouseParser::limitClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1247); - match(ClickHouseParser::LIMIT); - setState(1248); - limitExpr(); setState(1251); + match(ClickHouseParser::LIMIT); + setState(1252); + limitExpr(); + setState(1255); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::WITH) { - setState(1249); + setState(1253); match(ClickHouseParser::WITH); - setState(1250); + setState(1254); match(ClickHouseParser::TIES); } @@ -9788,7 +9737,6 @@ size_t ClickHouseParser::SettingsClauseContext::getRuleIndex() const { return ClickHouseParser::RuleSettingsClause; } - antlrcpp::Any ClickHouseParser::SettingsClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSettingsClause(this); @@ -9805,9 +9753,9 @@ ClickHouseParser::SettingsClauseContext* ClickHouseParser::settingsClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1253); + setState(1257); match(ClickHouseParser::SETTINGS); - setState(1254); + setState(1258); settingExprList(); } @@ -9867,7 +9815,6 @@ tree::TerminalNode* ClickHouseParser::JoinExprOpContext::LOCAL() { ClickHouseParser::JoinExprOpContext::JoinExprOpContext(JoinExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinExprOpContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinExprOp(this); @@ -9890,7 +9837,6 @@ ClickHouseParser::SampleClauseContext* ClickHouseParser::JoinExprTableContext::s ClickHouseParser::JoinExprTableContext::JoinExprTableContext(JoinExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinExprTableContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinExprTable(this); @@ -9913,7 +9859,6 @@ tree::TerminalNode* ClickHouseParser::JoinExprParensContext::RPAREN() { ClickHouseParser::JoinExprParensContext::JoinExprParensContext(JoinExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinExprParensContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinExprParens(this); @@ -9936,7 +9881,6 @@ ClickHouseParser::JoinOpCrossContext* ClickHouseParser::JoinExprCrossOpContext:: ClickHouseParser::JoinExprCrossOpContext::JoinExprCrossOpContext(JoinExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinExprCrossOpContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinExprCrossOp(this); @@ -9965,33 +9909,33 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1268); + setState(1272); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 159, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 160, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1257); + setState(1261); tableExpr(0); - setState(1259); + setState(1263); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 157, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 158, _ctx)) { case 1: { - setState(1258); + setState(1262); match(ClickHouseParser::FINAL); break; } } - setState(1262); + setState(1266); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 158, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 159, _ctx)) { case 1: { - setState(1261); + setState(1265); sampleClause(); break; } @@ -10004,38 +9948,38 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1264); + setState(1268); match(ClickHouseParser::LPAREN); - setState(1265); + setState(1269); joinExpr(0); - setState(1266); + setState(1270); match(ClickHouseParser::RPAREN); break; } } _ctx->stop = _input->LT(-1); - setState(1287); + setState(1291); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 163, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 164, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) triggerExitRuleEvent(); previousContext = _localctx; - setState(1285); + setState(1289); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 162, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 163, _ctx)) { case 1: { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleJoinExpr); - setState(1270); + setState(1274); if (!(precpred(_ctx, 3))) throw FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(1271); + setState(1275); joinOpCross(); - setState(1272); + setState(1276); joinExpr(4); break; } @@ -10044,17 +9988,17 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleJoinExpr); - setState(1274); + setState(1278); if (!(precpred(_ctx, 4))) throw FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(1276); + setState(1280); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL || _la == ClickHouseParser::LOCAL) { - setState(1275); + setState(1279); _la = _input->LA(1); if (!(_la == ClickHouseParser::GLOBAL @@ -10066,7 +10010,7 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { consume(); } } - setState(1279); + setState(1283); _errHandler->sync(this); _la = _input->LA(1); @@ -10075,28 +10019,28 @@ ClickHouseParser::JoinExprContext* ClickHouseParser::joinExpr(int precedence) { | (1ULL << (ClickHouseParser::ANTI - 4)) | (1ULL << (ClickHouseParser::ANY - 4)) | (1ULL << (ClickHouseParser::ASOF - 4)) - | (1ULL << (ClickHouseParser::FULL - 4)))) != 0) || ((((_la - 80) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 80)) & ((1ULL << (ClickHouseParser::INNER - 80)) - | (1ULL << (ClickHouseParser::LEFT - 80)) - | (1ULL << (ClickHouseParser::RIGHT - 80)) - | (1ULL << (ClickHouseParser::SEMI - 80)))) != 0)) { - setState(1278); + | (1ULL << (ClickHouseParser::FULL - 4)))) != 0) || ((((_la - 81) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 81)) & ((1ULL << (ClickHouseParser::INNER - 81)) + | (1ULL << (ClickHouseParser::LEFT - 81)) + | (1ULL << (ClickHouseParser::RIGHT - 81)) + | (1ULL << (ClickHouseParser::SEMI - 81)))) != 0)) { + setState(1282); joinOp(); } - setState(1281); + setState(1285); match(ClickHouseParser::JOIN); - setState(1282); + setState(1286); joinExpr(0); - setState(1283); + setState(1287); joinConstraintClause(); break; } } } - setState(1289); + setState(1293); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 163, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 164, _ctx); } } catch (RecognitionException &e) { @@ -10142,7 +10086,6 @@ tree::TerminalNode* ClickHouseParser::JoinOpFullContext::ANY() { ClickHouseParser::JoinOpFullContext::JoinOpFullContext(JoinOpContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinOpFullContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinOpFull(this); @@ -10169,7 +10112,6 @@ tree::TerminalNode* ClickHouseParser::JoinOpInnerContext::ASOF() { ClickHouseParser::JoinOpInnerContext::JoinOpInnerContext(JoinOpContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinOpInnerContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinOpInner(this); @@ -10212,7 +10154,6 @@ tree::TerminalNode* ClickHouseParser::JoinOpLeftRightContext::ASOF() { ClickHouseParser::JoinOpLeftRightContext::JoinOpLeftRightContext(JoinOpContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::JoinOpLeftRightContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinOpLeftRight(this); @@ -10228,17 +10169,17 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { exitRule(); }); try { - setState(1333); + setState(1337); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 177, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 178, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1299); + setState(1303); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 166, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 167, _ctx)) { case 1: { - setState(1291); + setState(1295); _errHandler->sync(this); _la = _input->LA(1); @@ -10246,7 +10187,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0)) { - setState(1290); + setState(1294); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10259,15 +10200,15 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1293); + setState(1297); match(ClickHouseParser::INNER); break; } case 2: { - setState(1294); + setState(1298); match(ClickHouseParser::INNER); - setState(1296); + setState(1300); _errHandler->sync(this); _la = _input->LA(1); @@ -10275,7 +10216,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0)) { - setState(1295); + setState(1299); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10292,7 +10233,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { } case 3: { - setState(1298); + setState(1302); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10314,11 +10255,11 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1315); + setState(1319); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 171, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 172, _ctx)) { case 1: { - setState(1302); + setState(1306); _errHandler->sync(this); _la = _input->LA(1); @@ -10327,7 +10268,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { | (1ULL << ClickHouseParser::ANTI) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0) || _la == ClickHouseParser::SEMI) { - setState(1301); + setState(1305); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10341,29 +10282,6 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1304); - _la = _input->LA(1); - if (!(_la == ClickHouseParser::LEFT - - || _la == ClickHouseParser::RIGHT)) { - _errHandler->recoverInline(this); - } - else { - _errHandler->reportMatch(this); - consume(); - } - setState(1306); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::OUTER) { - setState(1305); - match(ClickHouseParser::OUTER); - } - break; - } - - case 2: { setState(1308); _la = _input->LA(1); if (!(_la == ClickHouseParser::LEFT @@ -10383,7 +10301,30 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { setState(1309); match(ClickHouseParser::OUTER); } - setState(1313); + break; + } + + case 2: { + setState(1312); + _la = _input->LA(1); + if (!(_la == ClickHouseParser::LEFT + + || _la == ClickHouseParser::RIGHT)) { + _errHandler->recoverInline(this); + } + else { + _errHandler->reportMatch(this); + consume(); + } + setState(1314); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::OUTER) { + setState(1313); + match(ClickHouseParser::OUTER); + } + setState(1317); _errHandler->sync(this); _la = _input->LA(1); @@ -10392,7 +10333,7 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { | (1ULL << ClickHouseParser::ANTI) | (1ULL << ClickHouseParser::ANY) | (1ULL << ClickHouseParser::ASOF))) != 0) || _la == ClickHouseParser::SEMI) { - setState(1312); + setState(1316); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::ALL) @@ -10416,18 +10357,18 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1331); + setState(1335); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 176, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 177, _ctx)) { case 1: { - setState(1318); + setState(1322); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ALL || _la == ClickHouseParser::ANY) { - setState(1317); + setState(1321); _la = _input->LA(1); if (!(_la == ClickHouseParser::ALL @@ -10439,20 +10380,6 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { consume(); } } - setState(1320); - match(ClickHouseParser::FULL); - setState(1322); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::OUTER) { - setState(1321); - match(ClickHouseParser::OUTER); - } - break; - } - - case 2: { setState(1324); match(ClickHouseParser::FULL); setState(1326); @@ -10463,14 +10390,28 @@ ClickHouseParser::JoinOpContext* ClickHouseParser::joinOp() { setState(1325); match(ClickHouseParser::OUTER); } - setState(1329); + break; + } + + case 2: { + setState(1328); + match(ClickHouseParser::FULL); + setState(1330); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::OUTER) { + setState(1329); + match(ClickHouseParser::OUTER); + } + setState(1333); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ALL || _la == ClickHouseParser::ANY) { - setState(1328); + setState(1332); _la = _input->LA(1); if (!(_la == ClickHouseParser::ALL @@ -10532,7 +10473,6 @@ size_t ClickHouseParser::JoinOpCrossContext::getRuleIndex() const { return ClickHouseParser::RuleJoinOpCross; } - antlrcpp::Any ClickHouseParser::JoinOpCrossContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinOpCross(this); @@ -10549,21 +10489,21 @@ ClickHouseParser::JoinOpCrossContext* ClickHouseParser::joinOpCross() { exitRule(); }); try { - setState(1341); + setState(1345); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::CROSS: case ClickHouseParser::GLOBAL: case ClickHouseParser::LOCAL: { enterOuterAlt(_localctx, 1); - setState(1336); + setState(1340); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL || _la == ClickHouseParser::LOCAL) { - setState(1335); + setState(1339); _la = _input->LA(1); if (!(_la == ClickHouseParser::GLOBAL @@ -10575,16 +10515,16 @@ ClickHouseParser::JoinOpCrossContext* ClickHouseParser::joinOpCross() { consume(); } } - setState(1338); + setState(1342); match(ClickHouseParser::CROSS); - setState(1339); + setState(1343); match(ClickHouseParser::JOIN); break; } case ClickHouseParser::COMMA: { enterOuterAlt(_localctx, 2); - setState(1340); + setState(1344); match(ClickHouseParser::COMMA); break; } @@ -10634,7 +10574,6 @@ size_t ClickHouseParser::JoinConstraintClauseContext::getRuleIndex() const { return ClickHouseParser::RuleJoinConstraintClause; } - antlrcpp::Any ClickHouseParser::JoinConstraintClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitJoinConstraintClause(this); @@ -10650,36 +10589,36 @@ ClickHouseParser::JoinConstraintClauseContext* ClickHouseParser::joinConstraintC exitRule(); }); try { - setState(1352); + setState(1356); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 180, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 181, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1343); + setState(1347); match(ClickHouseParser::ON); - setState(1344); + setState(1348); columnExprList(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1345); + setState(1349); match(ClickHouseParser::USING); - setState(1346); + setState(1350); match(ClickHouseParser::LPAREN); - setState(1347); + setState(1351); columnExprList(); - setState(1348); + setState(1352); match(ClickHouseParser::RPAREN); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1350); + setState(1354); match(ClickHouseParser::USING); - setState(1351); + setState(1355); columnExprList(); break; } @@ -10723,7 +10662,6 @@ size_t ClickHouseParser::SampleClauseContext::getRuleIndex() const { return ClickHouseParser::RuleSampleClause; } - antlrcpp::Any ClickHouseParser::SampleClauseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSampleClause(this); @@ -10740,18 +10678,18 @@ ClickHouseParser::SampleClauseContext* ClickHouseParser::sampleClause() { }); try { enterOuterAlt(_localctx, 1); - setState(1354); - match(ClickHouseParser::SAMPLE); - setState(1355); - ratioExpr(); setState(1358); + match(ClickHouseParser::SAMPLE); + setState(1359); + ratioExpr(); + setState(1362); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 181, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 182, _ctx)) { case 1: { - setState(1356); + setState(1360); match(ClickHouseParser::OFFSET); - setState(1357); + setState(1361); ratioExpr(); break; } @@ -10795,7 +10733,6 @@ size_t ClickHouseParser::LimitExprContext::getRuleIndex() const { return ClickHouseParser::RuleLimitExpr; } - antlrcpp::Any ClickHouseParser::LimitExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLimitExpr(this); @@ -10813,14 +10750,14 @@ ClickHouseParser::LimitExprContext* ClickHouseParser::limitExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1360); + setState(1364); columnExpr(0); - setState(1363); + setState(1367); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::OFFSET || _la == ClickHouseParser::COMMA) { - setState(1361); + setState(1365); _la = _input->LA(1); if (!(_la == ClickHouseParser::OFFSET || _la == ClickHouseParser::COMMA)) { _errHandler->recoverInline(this); @@ -10829,7 +10766,7 @@ ClickHouseParser::LimitExprContext* ClickHouseParser::limitExpr() { _errHandler->reportMatch(this); consume(); } - setState(1362); + setState(1366); columnExpr(0); } @@ -10870,7 +10807,6 @@ size_t ClickHouseParser::OrderExprListContext::getRuleIndex() const { return ClickHouseParser::RuleOrderExprList; } - antlrcpp::Any ClickHouseParser::OrderExprListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitOrderExprList(this); @@ -10888,21 +10824,21 @@ ClickHouseParser::OrderExprListContext* ClickHouseParser::orderExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1365); + setState(1369); orderExpr(); - setState(1370); + setState(1374); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 183, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 184, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1366); + setState(1370); match(ClickHouseParser::COMMA); - setState(1367); + setState(1371); orderExpr(); } - setState(1372); + setState(1376); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 183, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 184, _ctx); } } @@ -10962,7 +10898,6 @@ size_t ClickHouseParser::OrderExprContext::getRuleIndex() const { return ClickHouseParser::RuleOrderExpr; } - antlrcpp::Any ClickHouseParser::OrderExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitOrderExpr(this); @@ -10980,41 +10915,19 @@ ClickHouseParser::OrderExprContext* ClickHouseParser::orderExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1373); + setState(1377); columnExpr(0); - setState(1375); - _errHandler->sync(this); - - switch (getInterpreter()->adaptivePredict(_input, 184, _ctx)) { - case 1: { - setState(1374); - _la = _input->LA(1); - if (!((((_la & ~ 0x3fULL) == 0) && - ((1ULL << _la) & ((1ULL << ClickHouseParser::ASCENDING) - | (1ULL << ClickHouseParser::DESC) - | (1ULL << ClickHouseParser::DESCENDING))) != 0))) { - _errHandler->recoverInline(this); - } - else { - _errHandler->reportMatch(this); - consume(); - } - break; - } - - } setState(1379); _errHandler->sync(this); switch (getInterpreter()->adaptivePredict(_input, 185, _ctx)) { case 1: { - setState(1377); - match(ClickHouseParser::NULLS); setState(1378); _la = _input->LA(1); - if (!(_la == ClickHouseParser::FIRST - - || _la == ClickHouseParser::LAST)) { + if (!((((_la & ~ 0x3fULL) == 0) && + ((1ULL << _la) & ((1ULL << ClickHouseParser::ASCENDING) + | (1ULL << ClickHouseParser::DESC) + | (1ULL << ClickHouseParser::DESCENDING))) != 0))) { _errHandler->recoverInline(this); } else { @@ -11031,8 +10944,30 @@ ClickHouseParser::OrderExprContext* ClickHouseParser::orderExpr() { switch (getInterpreter()->adaptivePredict(_input, 186, _ctx)) { case 1: { setState(1381); - match(ClickHouseParser::COLLATE); + match(ClickHouseParser::NULLS); setState(1382); + _la = _input->LA(1); + if (!(_la == ClickHouseParser::FIRST + + || _la == ClickHouseParser::LAST)) { + _errHandler->recoverInline(this); + } + else { + _errHandler->reportMatch(this); + consume(); + } + break; + } + + } + setState(1387); + _errHandler->sync(this); + + switch (getInterpreter()->adaptivePredict(_input, 187, _ctx)) { + case 1: { + setState(1385); + match(ClickHouseParser::COLLATE); + setState(1386); match(ClickHouseParser::STRING_LITERAL); break; } @@ -11072,7 +11007,6 @@ size_t ClickHouseParser::RatioExprContext::getRuleIndex() const { return ClickHouseParser::RuleRatioExpr; } - antlrcpp::Any ClickHouseParser::RatioExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitRatioExpr(this); @@ -11089,16 +11023,16 @@ ClickHouseParser::RatioExprContext* ClickHouseParser::ratioExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1385); + setState(1389); numberLiteral(); - setState(1388); + setState(1392); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 187, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 188, _ctx)) { case 1: { - setState(1386); + setState(1390); match(ClickHouseParser::SLASH); - setState(1387); + setState(1391); numberLiteral(); break; } @@ -11142,7 +11076,6 @@ size_t ClickHouseParser::SettingExprListContext::getRuleIndex() const { return ClickHouseParser::RuleSettingExprList; } - antlrcpp::Any ClickHouseParser::SettingExprListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSettingExprList(this); @@ -11160,21 +11093,21 @@ ClickHouseParser::SettingExprListContext* ClickHouseParser::settingExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1390); + setState(1394); settingExpr(); - setState(1395); + setState(1399); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 188, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 189, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1391); + setState(1395); match(ClickHouseParser::COMMA); - setState(1392); + setState(1396); settingExpr(); } - setState(1397); + setState(1401); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 188, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 189, _ctx); } } @@ -11210,7 +11143,6 @@ size_t ClickHouseParser::SettingExprContext::getRuleIndex() const { return ClickHouseParser::RuleSettingExpr; } - antlrcpp::Any ClickHouseParser::SettingExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSettingExpr(this); @@ -11227,11 +11159,11 @@ ClickHouseParser::SettingExprContext* ClickHouseParser::settingExpr() { }); try { enterOuterAlt(_localctx, 1); - setState(1398); + setState(1402); identifier(); - setState(1399); + setState(1403); match(ClickHouseParser::EQ_SINGLE); - setState(1400); + setState(1404); literal(); } @@ -11263,7 +11195,6 @@ size_t ClickHouseParser::SetStmtContext::getRuleIndex() const { return ClickHouseParser::RuleSetStmt; } - antlrcpp::Any ClickHouseParser::SetStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSetStmt(this); @@ -11280,9 +11211,9 @@ ClickHouseParser::SetStmtContext* ClickHouseParser::setStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1402); + setState(1406); match(ClickHouseParser::SET); - setState(1403); + setState(1407); settingExprList(); } @@ -11330,7 +11261,6 @@ ClickHouseParser::DatabaseIdentifierContext* ClickHouseParser::ShowCreateDatabas ClickHouseParser::ShowCreateDatabaseStmtContext::ShowCreateDatabaseStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowCreateDatabaseStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowCreateDatabaseStmt(this); @@ -11349,7 +11279,6 @@ tree::TerminalNode* ClickHouseParser::ShowDatabasesStmtContext::DATABASES() { ClickHouseParser::ShowDatabasesStmtContext::ShowDatabasesStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowDatabasesStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowDatabasesStmt(this); @@ -11380,7 +11309,6 @@ tree::TerminalNode* ClickHouseParser::ShowCreateTableStmtContext::TABLE() { ClickHouseParser::ShowCreateTableStmtContext::ShowCreateTableStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowCreateTableStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowCreateTableStmt(this); @@ -11431,7 +11359,6 @@ tree::TerminalNode* ClickHouseParser::ShowTablesStmtContext::IN() { ClickHouseParser::ShowTablesStmtContext::ShowTablesStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowTablesStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowTablesStmt(this); @@ -11458,7 +11385,6 @@ ClickHouseParser::DatabaseIdentifierContext* ClickHouseParser::ShowDictionariesS ClickHouseParser::ShowDictionariesStmtContext::ShowDictionariesStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowDictionariesStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowDictionariesStmt(this); @@ -11485,7 +11411,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::ShowCreateDictionary ClickHouseParser::ShowCreateDictionaryStmtContext::ShowCreateDictionaryStmtContext(ShowStmtContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ShowCreateDictionaryStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitShowCreateDictionaryStmt(this); @@ -11501,19 +11426,19 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { exitRule(); }); try { - setState(1447); + setState(1451); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 196, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 197, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1405); + setState(1409); match(ClickHouseParser::SHOW); - setState(1406); + setState(1410); match(ClickHouseParser::CREATE); - setState(1407); + setState(1411); match(ClickHouseParser::DATABASE); - setState(1408); + setState(1412); databaseIdentifier(); break; } @@ -11521,13 +11446,13 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1409); + setState(1413); match(ClickHouseParser::SHOW); - setState(1410); + setState(1414); match(ClickHouseParser::CREATE); - setState(1411); + setState(1415); match(ClickHouseParser::DICTIONARY); - setState(1412); + setState(1416); tableIdentifier(); break; } @@ -11535,33 +11460,33 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1413); + setState(1417); match(ClickHouseParser::SHOW); - setState(1414); + setState(1418); match(ClickHouseParser::CREATE); - setState(1416); + setState(1420); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 189, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 190, _ctx)) { case 1: { - setState(1415); + setState(1419); match(ClickHouseParser::TEMPORARY); break; } } - setState(1419); + setState(1423); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 190, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 191, _ctx)) { case 1: { - setState(1418); + setState(1422); match(ClickHouseParser::TABLE); break; } } - setState(1421); + setState(1425); tableIdentifier(); break; } @@ -11569,9 +11494,9 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(1422); + setState(1426); match(ClickHouseParser::SHOW); - setState(1423); + setState(1427); match(ClickHouseParser::DATABASES); break; } @@ -11579,18 +11504,18 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 5: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 5); - setState(1424); - match(ClickHouseParser::SHOW); - setState(1425); - match(ClickHouseParser::DICTIONARIES); setState(1428); + match(ClickHouseParser::SHOW); + setState(1429); + match(ClickHouseParser::DICTIONARIES); + setState(1432); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM) { - setState(1426); + setState(1430); match(ClickHouseParser::FROM); - setState(1427); + setState(1431); databaseIdentifier(); } break; @@ -11599,26 +11524,26 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { case 6: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 6); - setState(1430); + setState(1434); match(ClickHouseParser::SHOW); - setState(1432); + setState(1436); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TEMPORARY) { - setState(1431); + setState(1435); match(ClickHouseParser::TEMPORARY); } - setState(1434); + setState(1438); match(ClickHouseParser::TABLES); - setState(1437); + setState(1441); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FROM || _la == ClickHouseParser::IN) { - setState(1435); + setState(1439); _la = _input->LA(1); if (!(_la == ClickHouseParser::FROM @@ -11629,22 +11554,22 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { _errHandler->reportMatch(this); consume(); } - setState(1436); + setState(1440); databaseIdentifier(); } - setState(1442); + setState(1446); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::LIKE: { - setState(1439); + setState(1443); match(ClickHouseParser::LIKE); - setState(1440); + setState(1444); match(ClickHouseParser::STRING_LITERAL); break; } case ClickHouseParser::WHERE: { - setState(1441); + setState(1445); whereClause(); break; } @@ -11660,12 +11585,12 @@ ClickHouseParser::ShowStmtContext* ClickHouseParser::showStmt() { default: break; } - setState(1445); + setState(1449); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::LIMIT) { - setState(1444); + setState(1448); limitClause(); } break; @@ -11762,7 +11687,6 @@ size_t ClickHouseParser::SystemStmtContext::getRuleIndex() const { return ClickHouseParser::RuleSystemStmt; } - antlrcpp::Any ClickHouseParser::SystemStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitSystemStmt(this); @@ -11779,62 +11703,62 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { exitRule(); }); try { - setState(1483); + setState(1487); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 199, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 200, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1449); + setState(1453); match(ClickHouseParser::SYSTEM); - setState(1450); + setState(1454); match(ClickHouseParser::FLUSH); - setState(1451); + setState(1455); match(ClickHouseParser::DISTRIBUTED); - setState(1452); + setState(1456); tableIdentifier(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1453); + setState(1457); match(ClickHouseParser::SYSTEM); - setState(1454); + setState(1458); match(ClickHouseParser::FLUSH); - setState(1455); + setState(1459); match(ClickHouseParser::LOGS); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1456); + setState(1460); match(ClickHouseParser::SYSTEM); - setState(1457); + setState(1461); match(ClickHouseParser::RELOAD); - setState(1458); + setState(1462); match(ClickHouseParser::DICTIONARIES); break; } case 4: { enterOuterAlt(_localctx, 4); - setState(1459); + setState(1463); match(ClickHouseParser::SYSTEM); - setState(1460); + setState(1464); match(ClickHouseParser::RELOAD); - setState(1461); + setState(1465); match(ClickHouseParser::DICTIONARY); - setState(1462); + setState(1466); tableIdentifier(); break; } case 5: { enterOuterAlt(_localctx, 5); - setState(1463); + setState(1467); match(ClickHouseParser::SYSTEM); - setState(1464); + setState(1468); _la = _input->LA(1); if (!(_la == ClickHouseParser::START @@ -11845,34 +11769,34 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { _errHandler->reportMatch(this); consume(); } - setState(1472); + setState(1476); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DISTRIBUTED: { - setState(1465); + setState(1469); match(ClickHouseParser::DISTRIBUTED); - setState(1466); + setState(1470); match(ClickHouseParser::SENDS); break; } case ClickHouseParser::FETCHES: { - setState(1467); + setState(1471); match(ClickHouseParser::FETCHES); break; } case ClickHouseParser::MERGES: case ClickHouseParser::TTL: { - setState(1469); + setState(1473); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::TTL) { - setState(1468); + setState(1472); match(ClickHouseParser::TTL); } - setState(1471); + setState(1475); match(ClickHouseParser::MERGES); break; } @@ -11880,16 +11804,16 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { default: throw NoViableAltException(this); } - setState(1474); + setState(1478); tableIdentifier(); break; } case 6: { enterOuterAlt(_localctx, 6); - setState(1475); + setState(1479); match(ClickHouseParser::SYSTEM); - setState(1476); + setState(1480); _la = _input->LA(1); if (!(_la == ClickHouseParser::START @@ -11900,22 +11824,22 @@ ClickHouseParser::SystemStmtContext* ClickHouseParser::systemStmt() { _errHandler->reportMatch(this); consume(); } - setState(1477); + setState(1481); match(ClickHouseParser::REPLICATED); - setState(1478); + setState(1482); match(ClickHouseParser::SENDS); break; } case 7: { enterOuterAlt(_localctx, 7); - setState(1479); + setState(1483); match(ClickHouseParser::SYSTEM); - setState(1480); + setState(1484); match(ClickHouseParser::SYNC); - setState(1481); + setState(1485); match(ClickHouseParser::REPLICA); - setState(1482); + setState(1486); tableIdentifier(); break; } @@ -11971,7 +11895,6 @@ size_t ClickHouseParser::TruncateStmtContext::getRuleIndex() const { return ClickHouseParser::RuleTruncateStmt; } - antlrcpp::Any ClickHouseParser::TruncateStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTruncateStmt(this); @@ -11989,26 +11912,15 @@ ClickHouseParser::TruncateStmtContext* ClickHouseParser::truncateStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1485); + setState(1489); match(ClickHouseParser::TRUNCATE); - setState(1487); - _errHandler->sync(this); - - switch (getInterpreter()->adaptivePredict(_input, 200, _ctx)) { - case 1: { - setState(1486); - match(ClickHouseParser::TEMPORARY); - break; - } - - } - setState(1490); + setState(1491); _errHandler->sync(this); switch (getInterpreter()->adaptivePredict(_input, 201, _ctx)) { case 1: { - setState(1489); - match(ClickHouseParser::TABLE); + setState(1490); + match(ClickHouseParser::TEMPORARY); break; } @@ -12018,22 +11930,33 @@ ClickHouseParser::TruncateStmtContext* ClickHouseParser::truncateStmt() { switch (getInterpreter()->adaptivePredict(_input, 202, _ctx)) { case 1: { - setState(1492); - match(ClickHouseParser::IF); setState(1493); + match(ClickHouseParser::TABLE); + break; + } + + } + setState(1498); + _errHandler->sync(this); + + switch (getInterpreter()->adaptivePredict(_input, 203, _ctx)) { + case 1: { + setState(1496); + match(ClickHouseParser::IF); + setState(1497); match(ClickHouseParser::EXISTS); break; } } - setState(1496); + setState(1500); tableIdentifier(); - setState(1498); + setState(1502); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ON) { - setState(1497); + setState(1501); clusterClause(); } @@ -12066,7 +11989,6 @@ size_t ClickHouseParser::UseStmtContext::getRuleIndex() const { return ClickHouseParser::RuleUseStmt; } - antlrcpp::Any ClickHouseParser::UseStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitUseStmt(this); @@ -12083,9 +12005,9 @@ ClickHouseParser::UseStmtContext* ClickHouseParser::useStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1500); + setState(1504); match(ClickHouseParser::USE); - setState(1501); + setState(1505); databaseIdentifier(); } @@ -12129,7 +12051,6 @@ size_t ClickHouseParser::WatchStmtContext::getRuleIndex() const { return ClickHouseParser::RuleWatchStmt; } - antlrcpp::Any ClickHouseParser::WatchStmtContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitWatchStmt(this); @@ -12147,26 +12068,26 @@ ClickHouseParser::WatchStmtContext* ClickHouseParser::watchStmt() { }); try { enterOuterAlt(_localctx, 1); - setState(1503); + setState(1507); match(ClickHouseParser::WATCH); - setState(1504); + setState(1508); tableIdentifier(); - setState(1506); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::EVENTS) { - setState(1505); - match(ClickHouseParser::EVENTS); - } setState(1510); _errHandler->sync(this); _la = _input->LA(1); - if (_la == ClickHouseParser::LIMIT) { - setState(1508); - match(ClickHouseParser::LIMIT); + if (_la == ClickHouseParser::EVENTS) { setState(1509); + match(ClickHouseParser::EVENTS); + } + setState(1514); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::LIMIT) { + setState(1512); + match(ClickHouseParser::LIMIT); + setState(1513); match(ClickHouseParser::DECIMAL_LITERAL); } @@ -12231,7 +12152,6 @@ tree::TerminalNode* ClickHouseParser::ColumnTypeExprNestedContext::COMMA(size_t ClickHouseParser::ColumnTypeExprNestedContext::ColumnTypeExprNestedContext(ColumnTypeExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnTypeExprNestedContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnTypeExprNested(this); @@ -12258,7 +12178,6 @@ ClickHouseParser::ColumnExprListContext* ClickHouseParser::ColumnTypeExprParamCo ClickHouseParser::ColumnTypeExprParamContext::ColumnTypeExprParamContext(ColumnTypeExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnTypeExprParamContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnTypeExprParam(this); @@ -12273,7 +12192,6 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::ColumnTypeExprSimpleConte ClickHouseParser::ColumnTypeExprSimpleContext::ColumnTypeExprSimpleContext(ColumnTypeExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnTypeExprSimpleContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnTypeExprSimple(this); @@ -12312,7 +12230,6 @@ tree::TerminalNode* ClickHouseParser::ColumnTypeExprComplexContext::COMMA(size_t ClickHouseParser::ColumnTypeExprComplexContext::ColumnTypeExprComplexContext(ColumnTypeExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnTypeExprComplexContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnTypeExprComplex(this); @@ -12351,7 +12268,6 @@ tree::TerminalNode* ClickHouseParser::ColumnTypeExprEnumContext::COMMA(size_t i) ClickHouseParser::ColumnTypeExprEnumContext::ColumnTypeExprEnumContext(ColumnTypeExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnTypeExprEnumContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnTypeExprEnum(this); @@ -12367,13 +12283,13 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { exitRule(); }); try { - setState(1559); + setState(1563); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 210, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 211, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1512); + setState(1516); identifier(); break; } @@ -12381,29 +12297,29 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1513); + setState(1517); identifier(); - setState(1514); + setState(1518); match(ClickHouseParser::LPAREN); - setState(1515); + setState(1519); identifier(); - setState(1516); + setState(1520); columnTypeExpr(); - setState(1523); + setState(1527); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1517); + setState(1521); match(ClickHouseParser::COMMA); - setState(1518); + setState(1522); identifier(); - setState(1519); + setState(1523); columnTypeExpr(); - setState(1525); + setState(1529); _errHandler->sync(this); _la = _input->LA(1); } - setState(1526); + setState(1530); match(ClickHouseParser::RPAREN); break; } @@ -12411,25 +12327,25 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1528); + setState(1532); identifier(); - setState(1529); + setState(1533); match(ClickHouseParser::LPAREN); - setState(1530); + setState(1534); enumValue(); - setState(1535); + setState(1539); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1531); + setState(1535); match(ClickHouseParser::COMMA); - setState(1532); + setState(1536); enumValue(); - setState(1537); + setState(1541); _errHandler->sync(this); _la = _input->LA(1); } - setState(1538); + setState(1542); match(ClickHouseParser::RPAREN); break; } @@ -12437,25 +12353,25 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 4: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 4); - setState(1540); + setState(1544); identifier(); - setState(1541); + setState(1545); match(ClickHouseParser::LPAREN); - setState(1542); + setState(1546); columnTypeExpr(); - setState(1547); + setState(1551); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1543); + setState(1547); match(ClickHouseParser::COMMA); - setState(1544); + setState(1548); columnTypeExpr(); - setState(1549); + setState(1553); _errHandler->sync(this); _la = _input->LA(1); } - setState(1550); + setState(1554); match(ClickHouseParser::RPAREN); break; } @@ -12463,11 +12379,11 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { case 5: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 5); - setState(1552); + setState(1556); identifier(); - setState(1553); + setState(1557); match(ClickHouseParser::LPAREN); - setState(1555); + setState(1559); _errHandler->sync(this); _la = _input->LA(1); @@ -12483,6 +12399,7 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -12532,9 +12449,9 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -12569,8 +12486,8 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -12595,9 +12512,9 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -12658,16 +12575,16 @@ ClickHouseParser::ColumnTypeExprContext* ClickHouseParser::columnTypeExpr() { | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { - setState(1554); + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { + setState(1558); columnExprList(); } - setState(1557); + setState(1561); match(ClickHouseParser::RPAREN); break; } @@ -12711,7 +12628,6 @@ size_t ClickHouseParser::ColumnExprListContext::getRuleIndex() const { return ClickHouseParser::RuleColumnExprList; } - antlrcpp::Any ClickHouseParser::ColumnExprListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprList(this); @@ -12729,21 +12645,21 @@ ClickHouseParser::ColumnExprListContext* ClickHouseParser::columnExprList() { try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1561); + setState(1565); columnsExpr(); - setState(1566); + setState(1570); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 211, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 212, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { - setState(1562); + setState(1566); match(ClickHouseParser::COMMA); - setState(1563); + setState(1567); columnsExpr(); } - setState(1568); + setState(1572); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 211, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 212, _ctx); } } @@ -12779,7 +12695,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::ColumnsExprColumnContext: ClickHouseParser::ColumnsExprColumnContext::ColumnsExprColumnContext(ColumnsExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnsExprColumnContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnsExprColumn(this); @@ -12802,7 +12717,6 @@ tree::TerminalNode* ClickHouseParser::ColumnsExprAsteriskContext::DOT() { ClickHouseParser::ColumnsExprAsteriskContext::ColumnsExprAsteriskContext(ColumnsExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnsExprAsteriskContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnsExprAsterisk(this); @@ -12825,7 +12739,6 @@ tree::TerminalNode* ClickHouseParser::ColumnsExprSubqueryContext::RPAREN() { ClickHouseParser::ColumnsExprSubqueryContext::ColumnsExprSubqueryContext(ColumnsExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnsExprSubqueryContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnsExprSubquery(this); @@ -12841,13 +12754,13 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { exitRule(); }); try { - setState(1580); + setState(1584); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 213, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 214, _ctx)) { case 1: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 1); - setState(1572); + setState(1576); _errHandler->sync(this); _la = _input->LA(1); @@ -12863,6 +12776,7 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -12912,9 +12826,9 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -12948,8 +12862,8 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -12972,9 +12886,9 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -13030,12 +12944,12 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(1569); + setState(1573); tableIdentifier(); - setState(1570); + setState(1574); match(ClickHouseParser::DOT); } - setState(1574); + setState(1578); match(ClickHouseParser::ASTERISK); break; } @@ -13043,11 +12957,11 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { case 2: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 2); - setState(1575); + setState(1579); match(ClickHouseParser::LPAREN); - setState(1576); + setState(1580); selectUnionStmt(); - setState(1577); + setState(1581); match(ClickHouseParser::RPAREN); break; } @@ -13055,7 +12969,7 @@ ClickHouseParser::ColumnsExprContext* ClickHouseParser::columnsExpr() { case 3: { _localctx = dynamic_cast(_tracker.createInstance(_localctx)); enterOuterAlt(_localctx, 3); - setState(1579); + setState(1583); columnExpr(0); break; } @@ -13107,7 +13021,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprTernaryOpContext::COLON() { ClickHouseParser::ColumnExprTernaryOpContext::ColumnExprTernaryOpContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprTernaryOpContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprTernaryOp(this); @@ -13134,7 +13047,6 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::ColumnExprAliasContext::i ClickHouseParser::ColumnExprAliasContext::ColumnExprAliasContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprAliasContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprAlias(this); @@ -13169,7 +13081,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprExtractContext::RPAREN() { ClickHouseParser::ColumnExprExtractContext::ColumnExprExtractContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprExtractContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprExtract(this); @@ -13188,7 +13099,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::ColumnExprNegateContext:: ClickHouseParser::ColumnExprNegateContext::ColumnExprNegateContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprNegateContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprNegate(this); @@ -13211,7 +13121,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprSubqueryContext::RPAREN() { ClickHouseParser::ColumnExprSubqueryContext::ColumnExprSubqueryContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprSubqueryContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprSubquery(this); @@ -13226,7 +13135,6 @@ ClickHouseParser::LiteralContext* ClickHouseParser::ColumnExprLiteralContext::li ClickHouseParser::ColumnExprLiteralContext::ColumnExprLiteralContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprLiteralContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprLiteral(this); @@ -13249,7 +13157,6 @@ ClickHouseParser::ColumnExprListContext* ClickHouseParser::ColumnExprArrayContex ClickHouseParser::ColumnExprArrayContext::ColumnExprArrayContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprArrayContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprArray(this); @@ -13288,7 +13195,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprSubstringContext::FOR() { ClickHouseParser::ColumnExprSubstringContext::ColumnExprSubstringContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprSubstringContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprSubstring(this); @@ -13323,7 +13229,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprCastContext::RPAREN() { ClickHouseParser::ColumnExprCastContext::ColumnExprCastContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprCastContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprCast(this); @@ -13346,7 +13251,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprOrContext::OR() { ClickHouseParser::ColumnExprOrContext::ColumnExprOrContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprOrContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprOr(this); @@ -13377,7 +13281,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprPrecedence1Context::PERCENT() { ClickHouseParser::ColumnExprPrecedence1Context::ColumnExprPrecedence1Context(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprPrecedence1Context::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprPrecedence1(this); @@ -13408,7 +13311,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprPrecedence2Context::CONCAT() { ClickHouseParser::ColumnExprPrecedence2Context::ColumnExprPrecedence2Context(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprPrecedence2Context::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprPrecedence2(this); @@ -13475,7 +13377,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprPrecedence3Context::NOT() { ClickHouseParser::ColumnExprPrecedence3Context::ColumnExprPrecedence3Context(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprPrecedence3Context::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprPrecedence3(this); @@ -13498,7 +13399,6 @@ ClickHouseParser::IntervalContext* ClickHouseParser::ColumnExprIntervalContext:: ClickHouseParser::ColumnExprIntervalContext::ColumnExprIntervalContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprIntervalContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprInterval(this); @@ -13525,7 +13425,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprIsNullContext::NOT() { ClickHouseParser::ColumnExprIsNullContext::ColumnExprIsNullContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprIsNullContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprIsNull(this); @@ -13572,7 +13471,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprTrimContext::TRAILING() { ClickHouseParser::ColumnExprTrimContext::ColumnExprTrimContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprTrimContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprTrim(this); @@ -13595,7 +13493,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprTupleContext::RPAREN() { ClickHouseParser::ColumnExprTupleContext::ColumnExprTupleContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprTupleContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprTuple(this); @@ -13622,7 +13519,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprArrayAccessContext::RBRACKET() { ClickHouseParser::ColumnExprArrayAccessContext::ColumnExprArrayAccessContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprArrayAccessContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprArrayAccess(this); @@ -13653,7 +13549,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprBetweenContext::NOT() { ClickHouseParser::ColumnExprBetweenContext::ColumnExprBetweenContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprBetweenContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprBetween(this); @@ -13676,7 +13571,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprParensContext::RPAREN() { ClickHouseParser::ColumnExprParensContext::ColumnExprParensContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprParensContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprParens(this); @@ -13695,7 +13589,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprTimestampContext::STRING_LITERAL ClickHouseParser::ColumnExprTimestampContext::ColumnExprTimestampContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprTimestampContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprTimestamp(this); @@ -13718,7 +13611,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprAndContext::AND() { ClickHouseParser::ColumnExprAndContext::ColumnExprAndContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprAndContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprAnd(this); @@ -13741,7 +13633,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprTupleAccessContext::DECIMAL_LITE ClickHouseParser::ColumnExprTupleAccessContext::ColumnExprTupleAccessContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprTupleAccessContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprTupleAccess(this); @@ -13788,7 +13679,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprCaseContext::ELSE() { ClickHouseParser::ColumnExprCaseContext::ColumnExprCaseContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprCaseContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprCase(this); @@ -13807,7 +13697,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprDateContext::STRING_LITERAL() { ClickHouseParser::ColumnExprDateContext::ColumnExprDateContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprDateContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprDate(this); @@ -13826,7 +13715,6 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::ColumnExprNotContext::col ClickHouseParser::ColumnExprNotContext::ColumnExprNotContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprNotContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprNot(this); @@ -13841,7 +13729,6 @@ ClickHouseParser::ColumnIdentifierContext* ClickHouseParser::ColumnExprIdentifie ClickHouseParser::ColumnExprIdentifierContext::ColumnExprIdentifierContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprIdentifier(this); @@ -13884,7 +13771,6 @@ ClickHouseParser::ColumnExprListContext* ClickHouseParser::ColumnExprFunctionCon ClickHouseParser::ColumnExprFunctionContext::ColumnExprFunctionContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprFunctionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprFunction(this); @@ -13907,7 +13793,6 @@ tree::TerminalNode* ClickHouseParser::ColumnExprAsteriskContext::DOT() { ClickHouseParser::ColumnExprAsteriskContext::ColumnExprAsteriskContext(ColumnExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::ColumnExprAsteriskContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnExprAsterisk(this); @@ -13936,54 +13821,54 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1689); + setState(1693); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 224, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 225, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1583); + setState(1587); match(ClickHouseParser::CASE); - setState(1585); + setState(1589); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 214, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 215, _ctx)) { case 1: { - setState(1584); + setState(1588); columnExpr(0); break; } } - setState(1592); + setState(1596); _errHandler->sync(this); _la = _input->LA(1); do { - setState(1587); + setState(1591); match(ClickHouseParser::WHEN); - setState(1588); + setState(1592); columnExpr(0); - setState(1589); + setState(1593); match(ClickHouseParser::THEN); - setState(1590); + setState(1594); columnExpr(0); - setState(1594); + setState(1598); _errHandler->sync(this); _la = _input->LA(1); } while (_la == ClickHouseParser::WHEN); - setState(1598); + setState(1602); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::ELSE) { - setState(1596); + setState(1600); match(ClickHouseParser::ELSE); - setState(1597); + setState(1601); columnExpr(0); } - setState(1600); + setState(1604); match(ClickHouseParser::END); break; } @@ -13992,17 +13877,17 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1602); - match(ClickHouseParser::CAST); - setState(1603); - match(ClickHouseParser::LPAREN); - setState(1604); - columnExpr(0); - setState(1605); - match(ClickHouseParser::AS); setState(1606); - columnTypeExpr(); + match(ClickHouseParser::CAST); setState(1607); + match(ClickHouseParser::LPAREN); + setState(1608); + columnExpr(0); + setState(1609); + match(ClickHouseParser::AS); + setState(1610); + columnTypeExpr(); + setState(1611); match(ClickHouseParser::RPAREN); break; } @@ -14011,9 +13896,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1609); + setState(1613); match(ClickHouseParser::DATE); - setState(1610); + setState(1614); match(ClickHouseParser::STRING_LITERAL); break; } @@ -14022,17 +13907,17 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1611); - match(ClickHouseParser::EXTRACT); - setState(1612); - match(ClickHouseParser::LPAREN); - setState(1613); - interval(); - setState(1614); - match(ClickHouseParser::FROM); setState(1615); - columnExpr(0); + match(ClickHouseParser::EXTRACT); setState(1616); + match(ClickHouseParser::LPAREN); + setState(1617); + interval(); + setState(1618); + match(ClickHouseParser::FROM); + setState(1619); + columnExpr(0); + setState(1620); match(ClickHouseParser::RPAREN); break; } @@ -14041,11 +13926,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1618); + setState(1622); match(ClickHouseParser::INTERVAL); - setState(1619); + setState(1623); columnExpr(0); - setState(1620); + setState(1624); interval(); break; } @@ -14054,27 +13939,27 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1622); - match(ClickHouseParser::SUBSTRING); - setState(1623); - match(ClickHouseParser::LPAREN); - setState(1624); - columnExpr(0); - setState(1625); - match(ClickHouseParser::FROM); setState(1626); + match(ClickHouseParser::SUBSTRING); + setState(1627); + match(ClickHouseParser::LPAREN); + setState(1628); columnExpr(0); setState(1629); + match(ClickHouseParser::FROM); + setState(1630); + columnExpr(0); + setState(1633); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::FOR) { - setState(1627); + setState(1631); match(ClickHouseParser::FOR); - setState(1628); + setState(1632); columnExpr(0); } - setState(1631); + setState(1635); match(ClickHouseParser::RPAREN); break; } @@ -14083,9 +13968,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1633); + setState(1637); match(ClickHouseParser::TIMESTAMP); - setState(1634); + setState(1638); match(ClickHouseParser::STRING_LITERAL); break; } @@ -14094,11 +13979,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1635); + setState(1639); match(ClickHouseParser::TRIM); - setState(1636); + setState(1640); match(ClickHouseParser::LPAREN); - setState(1637); + setState(1641); _la = _input->LA(1); if (!(_la == ClickHouseParser::BOTH || _la == ClickHouseParser::LEADING || _la == ClickHouseParser::TRAILING)) { _errHandler->recoverInline(this); @@ -14107,13 +13992,13 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _errHandler->reportMatch(this); consume(); } - setState(1638); + setState(1642); match(ClickHouseParser::STRING_LITERAL); - setState(1639); + setState(1643); match(ClickHouseParser::FROM); - setState(1640); + setState(1644); columnExpr(0); - setState(1641); + setState(1645); match(ClickHouseParser::RPAREN); break; } @@ -14122,16 +14007,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1643); + setState(1647); identifier(); - setState(1649); + setState(1653); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 219, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 220, _ctx)) { case 1: { - setState(1644); + setState(1648); match(ClickHouseParser::LPAREN); - setState(1646); + setState(1650); _errHandler->sync(this); _la = _input->LA(1); @@ -14147,6 +14032,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -14196,9 +14082,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -14233,8 +14119,8 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -14259,9 +14145,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -14322,35 +14208,35 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { - setState(1645); + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { + setState(1649); columnExprList(); } - setState(1648); + setState(1652); match(ClickHouseParser::RPAREN); break; } } - setState(1651); + setState(1655); match(ClickHouseParser::LPAREN); - setState(1653); + setState(1657); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 220, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 221, _ctx)) { case 1: { - setState(1652); + setState(1656); match(ClickHouseParser::DISTINCT); break; } } - setState(1656); + setState(1660); _errHandler->sync(this); _la = _input->LA(1); @@ -14366,6 +14252,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -14415,9 +14302,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -14452,8 +14339,8 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -14478,9 +14365,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -14541,16 +14428,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { - setState(1655); + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { + setState(1659); columnArgList(); } - setState(1658); + setState(1662); match(ClickHouseParser::RPAREN); break; } @@ -14559,7 +14446,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1660); + setState(1664); literal(); break; } @@ -14568,9 +14455,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1661); + setState(1665); match(ClickHouseParser::DASH); - setState(1662); + setState(1666); columnExpr(17); break; } @@ -14579,9 +14466,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1663); + setState(1667); match(ClickHouseParser::NOT); - setState(1664); + setState(1668); columnExpr(12); break; } @@ -14590,7 +14477,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1668); + setState(1672); _errHandler->sync(this); _la = _input->LA(1); @@ -14606,6 +14493,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -14655,9 +14543,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -14691,8 +14579,8 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -14715,9 +14603,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -14773,12 +14661,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::JSON_FALSE - 128)) | (1ULL << (ClickHouseParser::JSON_TRUE - 128)) | (1ULL << (ClickHouseParser::IDENTIFIER - 128)))) != 0)) { - setState(1665); + setState(1669); tableIdentifier(); - setState(1666); + setState(1670); match(ClickHouseParser::DOT); } - setState(1670); + setState(1674); match(ClickHouseParser::ASTERISK); break; } @@ -14787,11 +14675,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1671); + setState(1675); match(ClickHouseParser::LPAREN); - setState(1672); + setState(1676); selectUnionStmt(); - setState(1673); + setState(1677); match(ClickHouseParser::RPAREN); break; } @@ -14800,11 +14688,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1675); + setState(1679); match(ClickHouseParser::LPAREN); - setState(1676); + setState(1680); columnExpr(0); - setState(1677); + setState(1681); match(ClickHouseParser::RPAREN); break; } @@ -14813,11 +14701,11 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1679); + setState(1683); match(ClickHouseParser::LPAREN); - setState(1680); + setState(1684); columnExprList(); - setState(1681); + setState(1685); match(ClickHouseParser::RPAREN); break; } @@ -14826,9 +14714,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1683); + setState(1687); match(ClickHouseParser::LBRACKET); - setState(1685); + setState(1689); _errHandler->sync(this); _la = _input->LA(1); @@ -14844,6 +14732,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -14893,9 +14782,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -14930,8 +14819,8 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -14956,9 +14845,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -15019,16 +14908,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)) - | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::LBRACKET - 196)) - | (1ULL << (ClickHouseParser::LPAREN - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { - setState(1684); + | (1ULL << (ClickHouseParser::ASTERISK - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::LBRACKET - 197)) + | (1ULL << (ClickHouseParser::LPAREN - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { + setState(1688); columnExprList(); } - setState(1687); + setState(1691); match(ClickHouseParser::RBRACKET); break; } @@ -15037,44 +14926,44 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1688); + setState(1692); columnIdentifier(); break; } } _ctx->stop = _input->LT(-1); - setState(1762); + setState(1766); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 233, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 234, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) triggerExitRuleEvent(); previousContext = _localctx; - setState(1760); + setState(1764); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 232, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 233, _ctx)) { case 1: { auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1691); + setState(1695); if (!(precpred(_ctx, 16))) throw FailedPredicateException(this, "precpred(_ctx, 16)"); - setState(1692); + setState(1696); _la = _input->LA(1); - if (!(((((_la - 190) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 190)) & ((1ULL << (ClickHouseParser::ASTERISK - 190)) - | (1ULL << (ClickHouseParser::PERCENT - 190)) - | (1ULL << (ClickHouseParser::SLASH - 190)))) != 0))) { + if (!(((((_la - 191) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 191)) & ((1ULL << (ClickHouseParser::ASTERISK - 191)) + | (1ULL << (ClickHouseParser::PERCENT - 191)) + | (1ULL << (ClickHouseParser::SLASH - 191)))) != 0))) { _errHandler->recoverInline(this); } else { _errHandler->reportMatch(this); consume(); } - setState(1693); + setState(1697); columnExpr(17); break; } @@ -15083,22 +14972,22 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1694); + setState(1698); if (!(precpred(_ctx, 15))) throw FailedPredicateException(this, "precpred(_ctx, 15)"); - setState(1695); + setState(1699); _la = _input->LA(1); - if (!(((((_la - 195) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 195)) & ((1ULL << (ClickHouseParser::CONCAT - 195)) - | (1ULL << (ClickHouseParser::DASH - 195)) - | (1ULL << (ClickHouseParser::PLUS - 195)))) != 0))) { + if (!(((((_la - 196) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::CONCAT - 196)) + | (1ULL << (ClickHouseParser::DASH - 196)) + | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0))) { _errHandler->recoverInline(this); } else { _errHandler->reportMatch(this); consume(); } - setState(1696); + setState(1700); columnExpr(16); break; } @@ -15107,77 +14996,63 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1697); + setState(1701); if (!(precpred(_ctx, 14))) throw FailedPredicateException(this, "precpred(_ctx, 14)"); - setState(1716); + setState(1720); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 228, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 229, _ctx)) { case 1: { - setState(1698); + setState(1702); match(ClickHouseParser::EQ_DOUBLE); break; } case 2: { - setState(1699); + setState(1703); match(ClickHouseParser::EQ_SINGLE); break; } case 3: { - setState(1700); + setState(1704); match(ClickHouseParser::NOT_EQ); break; } case 4: { - setState(1701); + setState(1705); match(ClickHouseParser::LE); break; } case 5: { - setState(1702); + setState(1706); match(ClickHouseParser::GE); break; } case 6: { - setState(1703); + setState(1707); match(ClickHouseParser::LT); break; } case 7: { - setState(1704); + setState(1708); match(ClickHouseParser::GT); break; } case 8: { - setState(1706); + setState(1710); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::GLOBAL) { - setState(1705); + setState(1709); match(ClickHouseParser::GLOBAL); } - setState(1709); - _errHandler->sync(this); - - _la = _input->LA(1); - if (_la == ClickHouseParser::NOT) { - setState(1708); - match(ClickHouseParser::NOT); - } - setState(1711); - match(ClickHouseParser::IN); - break; - } - - case 9: { setState(1713); _errHandler->sync(this); @@ -15187,6 +15062,20 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence match(ClickHouseParser::NOT); } setState(1715); + match(ClickHouseParser::IN); + break; + } + + case 9: { + setState(1717); + _errHandler->sync(this); + + _la = _input->LA(1); + if (_la == ClickHouseParser::NOT) { + setState(1716); + match(ClickHouseParser::NOT); + } + setState(1719); _la = _input->LA(1); if (!(_la == ClickHouseParser::ILIKE @@ -15201,7 +15090,7 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence } } - setState(1718); + setState(1722); columnExpr(15); break; } @@ -15210,12 +15099,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1719); + setState(1723); if (!(precpred(_ctx, 11))) throw FailedPredicateException(this, "precpred(_ctx, 11)"); - setState(1720); + setState(1724); match(ClickHouseParser::AND); - setState(1721); + setState(1725); columnExpr(12); break; } @@ -15224,12 +15113,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1722); + setState(1726); if (!(precpred(_ctx, 10))) throw FailedPredicateException(this, "precpred(_ctx, 10)"); - setState(1723); + setState(1727); match(ClickHouseParser::OR); - setState(1724); + setState(1728); columnExpr(11); break; } @@ -15238,24 +15127,24 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1725); + setState(1729); if (!(precpred(_ctx, 9))) throw FailedPredicateException(this, "precpred(_ctx, 9)"); - setState(1727); + setState(1731); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1726); + setState(1730); match(ClickHouseParser::NOT); } - setState(1729); + setState(1733); match(ClickHouseParser::BETWEEN); - setState(1730); + setState(1734); columnExpr(0); - setState(1731); + setState(1735); match(ClickHouseParser::AND); - setState(1732); + setState(1736); columnExpr(10); break; } @@ -15264,16 +15153,16 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1734); + setState(1738); if (!(precpred(_ctx, 8))) throw FailedPredicateException(this, "precpred(_ctx, 8)"); - setState(1735); + setState(1739); match(ClickHouseParser::QUERY); - setState(1736); + setState(1740); columnExpr(0); - setState(1737); + setState(1741); match(ClickHouseParser::COLON); - setState(1738); + setState(1742); columnExpr(8); break; } @@ -15282,14 +15171,14 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1740); + setState(1744); if (!(precpred(_ctx, 19))) throw FailedPredicateException(this, "precpred(_ctx, 19)"); - setState(1741); + setState(1745); match(ClickHouseParser::LBRACKET); - setState(1742); + setState(1746); columnExpr(0); - setState(1743); + setState(1747); match(ClickHouseParser::RBRACKET); break; } @@ -15298,12 +15187,12 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1745); + setState(1749); if (!(precpred(_ctx, 18))) throw FailedPredicateException(this, "precpred(_ctx, 18)"); - setState(1746); + setState(1750); match(ClickHouseParser::DOT); - setState(1747); + setState(1751); match(ClickHouseParser::DECIMAL_LITERAL); break; } @@ -15312,20 +15201,20 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1748); + setState(1752); if (!(precpred(_ctx, 13))) throw FailedPredicateException(this, "precpred(_ctx, 13)"); - setState(1749); + setState(1753); match(ClickHouseParser::IS); - setState(1751); + setState(1755); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::NOT) { - setState(1750); + setState(1754); match(ClickHouseParser::NOT); } - setState(1753); + setState(1757); match(ClickHouseParser::NULL_SQL); break; } @@ -15334,10 +15223,10 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleColumnExpr); - setState(1754); + setState(1758); if (!(precpred(_ctx, 7))) throw FailedPredicateException(this, "precpred(_ctx, 7)"); - setState(1758); + setState(1762); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DATE: @@ -15345,15 +15234,15 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence case ClickHouseParser::ID: case ClickHouseParser::KEY: case ClickHouseParser::IDENTIFIER: { - setState(1755); + setState(1759); alias(); break; } case ClickHouseParser::AS: { - setState(1756); + setState(1760); match(ClickHouseParser::AS); - setState(1757); + setState(1761); identifier(); break; } @@ -15366,9 +15255,9 @@ ClickHouseParser::ColumnExprContext* ClickHouseParser::columnExpr(int precedence } } - setState(1764); + setState(1768); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 233, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 234, _ctx); } } catch (RecognitionException &e) { @@ -15406,7 +15295,6 @@ size_t ClickHouseParser::ColumnArgListContext::getRuleIndex() const { return ClickHouseParser::RuleColumnArgList; } - antlrcpp::Any ClickHouseParser::ColumnArgListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnArgList(this); @@ -15424,17 +15312,17 @@ ClickHouseParser::ColumnArgListContext* ClickHouseParser::columnArgList() { }); try { enterOuterAlt(_localctx, 1); - setState(1765); + setState(1769); columnArgExpr(); - setState(1770); + setState(1774); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1766); + setState(1770); match(ClickHouseParser::COMMA); - setState(1767); + setState(1771); columnArgExpr(); - setState(1772); + setState(1776); _errHandler->sync(this); _la = _input->LA(1); } @@ -15468,7 +15356,6 @@ size_t ClickHouseParser::ColumnArgExprContext::getRuleIndex() const { return ClickHouseParser::RuleColumnArgExpr; } - antlrcpp::Any ClickHouseParser::ColumnArgExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnArgExpr(this); @@ -15484,19 +15371,19 @@ ClickHouseParser::ColumnArgExprContext* ClickHouseParser::columnArgExpr() { exitRule(); }); try { - setState(1775); + setState(1779); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 235, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 236, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1773); + setState(1777); columnLambdaExpr(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1774); + setState(1778); columnExpr(0); break; } @@ -15556,7 +15443,6 @@ size_t ClickHouseParser::ColumnLambdaExprContext::getRuleIndex() const { return ClickHouseParser::RuleColumnLambdaExpr; } - antlrcpp::Any ClickHouseParser::ColumnLambdaExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnLambdaExpr(this); @@ -15574,27 +15460,27 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() }); try { enterOuterAlt(_localctx, 1); - setState(1796); + setState(1800); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::LPAREN: { - setState(1777); + setState(1781); match(ClickHouseParser::LPAREN); - setState(1778); + setState(1782); identifier(); - setState(1783); + setState(1787); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1779); + setState(1783); match(ClickHouseParser::COMMA); - setState(1780); + setState(1784); identifier(); - setState(1785); + setState(1789); _errHandler->sync(this); _la = _input->LA(1); } - setState(1786); + setState(1790); match(ClickHouseParser::RPAREN); break; } @@ -15610,6 +15496,7 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() case ClickHouseParser::AS: case ClickHouseParser::ASCENDING: case ClickHouseParser::ASOF: + case ClickHouseParser::AST: case ClickHouseParser::ASYNC: case ClickHouseParser::ATTACH: case ClickHouseParser::BETWEEN: @@ -15695,8 +15582,8 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() case ClickHouseParser::LIVE: case ClickHouseParser::LOCAL: case ClickHouseParser::LOGS: - case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MATERIALIZE: + case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MAX: case ClickHouseParser::MERGES: case ClickHouseParser::MIN: @@ -15777,17 +15664,17 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { - setState(1788); + setState(1792); identifier(); - setState(1793); + setState(1797); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1789); + setState(1793); match(ClickHouseParser::COMMA); - setState(1790); + setState(1794); identifier(); - setState(1795); + setState(1799); _errHandler->sync(this); _la = _input->LA(1); } @@ -15797,9 +15684,9 @@ ClickHouseParser::ColumnLambdaExprContext* ClickHouseParser::columnLambdaExpr() default: throw NoViableAltException(this); } - setState(1798); + setState(1802); match(ClickHouseParser::ARROW); - setState(1799); + setState(1803); columnExpr(0); } @@ -15835,7 +15722,6 @@ size_t ClickHouseParser::ColumnIdentifierContext::getRuleIndex() const { return ClickHouseParser::RuleColumnIdentifier; } - antlrcpp::Any ClickHouseParser::ColumnIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitColumnIdentifier(this); @@ -15852,20 +15738,20 @@ ClickHouseParser::ColumnIdentifierContext* ClickHouseParser::columnIdentifier() }); try { enterOuterAlt(_localctx, 1); - setState(1804); + setState(1808); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 239, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 240, _ctx)) { case 1: { - setState(1801); + setState(1805); tableIdentifier(); - setState(1802); + setState(1806); match(ClickHouseParser::DOT); break; } } - setState(1806); + setState(1810); nestedIdentifier(); } @@ -15901,7 +15787,6 @@ size_t ClickHouseParser::NestedIdentifierContext::getRuleIndex() const { return ClickHouseParser::RuleNestedIdentifier; } - antlrcpp::Any ClickHouseParser::NestedIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitNestedIdentifier(this); @@ -15918,16 +15803,16 @@ ClickHouseParser::NestedIdentifierContext* ClickHouseParser::nestedIdentifier() }); try { enterOuterAlt(_localctx, 1); - setState(1808); + setState(1812); identifier(); - setState(1811); + setState(1815); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 240, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 241, _ctx)) { case 1: { - setState(1809); + setState(1813); match(ClickHouseParser::DOT); - setState(1810); + setState(1814); identifier(); break; } @@ -15967,7 +15852,6 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::TableExprIdentifierC ClickHouseParser::TableExprIdentifierContext::TableExprIdentifierContext(TableExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableExprIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableExprIdentifier(this); @@ -15990,7 +15874,6 @@ tree::TerminalNode* ClickHouseParser::TableExprSubqueryContext::RPAREN() { ClickHouseParser::TableExprSubqueryContext::TableExprSubqueryContext(TableExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableExprSubqueryContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableExprSubquery(this); @@ -16017,7 +15900,6 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::TableExprAliasContext::id ClickHouseParser::TableExprAliasContext::TableExprAliasContext(TableExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableExprAliasContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableExprAlias(this); @@ -16032,7 +15914,6 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::TableExprFunctionC ClickHouseParser::TableExprFunctionContext::TableExprFunctionContext(TableExprContext *ctx) { copyFrom(ctx); } - antlrcpp::Any ClickHouseParser::TableExprFunctionContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableExprFunction(this); @@ -16061,15 +15942,15 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) try { size_t alt; enterOuterAlt(_localctx, 1); - setState(1820); + setState(1824); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 241, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 242, _ctx)) { case 1: { _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1814); + setState(1818); tableIdentifier(); break; } @@ -16078,7 +15959,7 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1815); + setState(1819); tableFunctionExpr(); break; } @@ -16087,20 +15968,20 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) _localctx = _tracker.createInstance(_localctx); _ctx = _localctx; previousContext = _localctx; - setState(1816); + setState(1820); match(ClickHouseParser::LPAREN); - setState(1817); + setState(1821); selectUnionStmt(); - setState(1818); + setState(1822); match(ClickHouseParser::RPAREN); break; } } _ctx->stop = _input->LT(-1); - setState(1830); + setState(1834); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 243, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 244, _ctx); while (alt != 2 && alt != atn::ATN::INVALID_ALT_NUMBER) { if (alt == 1) { if (!_parseListeners.empty()) @@ -16109,10 +15990,10 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) auto newContext = _tracker.createInstance(_tracker.createInstance(parentContext, parentState)); _localctx = newContext; pushNewRecursionContext(newContext, startState, RuleTableExpr); - setState(1822); + setState(1826); if (!(precpred(_ctx, 1))) throw FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(1826); + setState(1830); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::DATE: @@ -16120,15 +16001,15 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) case ClickHouseParser::ID: case ClickHouseParser::KEY: case ClickHouseParser::IDENTIFIER: { - setState(1823); + setState(1827); alias(); break; } case ClickHouseParser::AS: { - setState(1824); + setState(1828); match(ClickHouseParser::AS); - setState(1825); + setState(1829); identifier(); break; } @@ -16137,9 +16018,9 @@ ClickHouseParser::TableExprContext* ClickHouseParser::tableExpr(int precedence) throw NoViableAltException(this); } } - setState(1832); + setState(1836); _errHandler->sync(this); - alt = getInterpreter()->adaptivePredict(_input, 243, _ctx); + alt = getInterpreter()->adaptivePredict(_input, 244, _ctx); } } catch (RecognitionException &e) { @@ -16177,7 +16058,6 @@ size_t ClickHouseParser::TableFunctionExprContext::getRuleIndex() const { return ClickHouseParser::RuleTableFunctionExpr; } - antlrcpp::Any ClickHouseParser::TableFunctionExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableFunctionExpr(this); @@ -16195,11 +16075,11 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( }); try { enterOuterAlt(_localctx, 1); - setState(1833); + setState(1837); identifier(); - setState(1834); + setState(1838); match(ClickHouseParser::LPAREN); - setState(1836); + setState(1840); _errHandler->sync(this); _la = _input->LA(1); @@ -16215,6 +16095,7 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -16264,9 +16145,9 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -16301,8 +16182,8 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -16327,9 +16208,9 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::QUARTER - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::QUARTER - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -16389,14 +16270,14 @@ ClickHouseParser::TableFunctionExprContext* ClickHouseParser::tableFunctionExpr( | (1ULL << (ClickHouseParser::OCTAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::DECIMAL_LITERAL - 128)) | (1ULL << (ClickHouseParser::HEXADECIMAL_LITERAL - 128)) - | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)))) != 0) || ((((_la - 196) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 196)) & ((1ULL << (ClickHouseParser::DASH - 196)) - | (1ULL << (ClickHouseParser::DOT - 196)) - | (1ULL << (ClickHouseParser::PLUS - 196)))) != 0)) { - setState(1835); + | (1ULL << (ClickHouseParser::STRING_LITERAL - 128)))) != 0) || ((((_la - 197) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 197)) & ((1ULL << (ClickHouseParser::DASH - 197)) + | (1ULL << (ClickHouseParser::DOT - 197)) + | (1ULL << (ClickHouseParser::PLUS - 197)))) != 0)) { + setState(1839); tableArgList(); } - setState(1838); + setState(1842); match(ClickHouseParser::RPAREN); } @@ -16432,7 +16313,6 @@ size_t ClickHouseParser::TableIdentifierContext::getRuleIndex() const { return ClickHouseParser::RuleTableIdentifier; } - antlrcpp::Any ClickHouseParser::TableIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableIdentifier(this); @@ -16449,20 +16329,20 @@ ClickHouseParser::TableIdentifierContext* ClickHouseParser::tableIdentifier() { }); try { enterOuterAlt(_localctx, 1); - setState(1843); + setState(1847); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 245, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 246, _ctx)) { case 1: { - setState(1840); + setState(1844); databaseIdentifier(); - setState(1841); + setState(1845); match(ClickHouseParser::DOT); break; } } - setState(1845); + setState(1849); identifier(); } @@ -16502,7 +16382,6 @@ size_t ClickHouseParser::TableArgListContext::getRuleIndex() const { return ClickHouseParser::RuleTableArgList; } - antlrcpp::Any ClickHouseParser::TableArgListContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableArgList(this); @@ -16520,17 +16399,17 @@ ClickHouseParser::TableArgListContext* ClickHouseParser::tableArgList() { }); try { enterOuterAlt(_localctx, 1); - setState(1847); + setState(1851); tableArgExpr(); - setState(1852); + setState(1856); _errHandler->sync(this); _la = _input->LA(1); while (_la == ClickHouseParser::COMMA) { - setState(1848); + setState(1852); match(ClickHouseParser::COMMA); - setState(1849); + setState(1853); tableArgExpr(); - setState(1854); + setState(1858); _errHandler->sync(this); _la = _input->LA(1); } @@ -16551,8 +16430,8 @@ ClickHouseParser::TableArgExprContext::TableArgExprContext(ParserRuleContext *pa : ParserRuleContext(parent, invokingState) { } -ClickHouseParser::TableIdentifierContext* ClickHouseParser::TableArgExprContext::tableIdentifier() { - return getRuleContext(0); +ClickHouseParser::IdentifierContext* ClickHouseParser::TableArgExprContext::identifier() { + return getRuleContext(0); } ClickHouseParser::TableFunctionExprContext* ClickHouseParser::TableArgExprContext::tableFunctionExpr() { @@ -16568,7 +16447,6 @@ size_t ClickHouseParser::TableArgExprContext::getRuleIndex() const { return ClickHouseParser::RuleTableArgExpr; } - antlrcpp::Any ClickHouseParser::TableArgExprContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitTableArgExpr(this); @@ -16584,26 +16462,26 @@ ClickHouseParser::TableArgExprContext* ClickHouseParser::tableArgExpr() { exitRule(); }); try { - setState(1858); + setState(1862); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 247, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 248, _ctx)) { case 1: { enterOuterAlt(_localctx, 1); - setState(1855); - tableIdentifier(); + setState(1859); + identifier(); break; } case 2: { enterOuterAlt(_localctx, 2); - setState(1856); + setState(1860); tableFunctionExpr(); break; } case 3: { enterOuterAlt(_localctx, 3); - setState(1857); + setState(1861); literal(); break; } @@ -16635,7 +16513,6 @@ size_t ClickHouseParser::DatabaseIdentifierContext::getRuleIndex() const { return ClickHouseParser::RuleDatabaseIdentifier; } - antlrcpp::Any ClickHouseParser::DatabaseIdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitDatabaseIdentifier(this); @@ -16652,7 +16529,7 @@ ClickHouseParser::DatabaseIdentifierContext* ClickHouseParser::databaseIdentifie }); try { enterOuterAlt(_localctx, 1); - setState(1860); + setState(1864); identifier(); } @@ -16696,7 +16573,6 @@ size_t ClickHouseParser::FloatingLiteralContext::getRuleIndex() const { return ClickHouseParser::RuleFloatingLiteral; } - antlrcpp::Any ClickHouseParser::FloatingLiteralContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitFloatingLiteral(this); @@ -16713,21 +16589,21 @@ ClickHouseParser::FloatingLiteralContext* ClickHouseParser::floatingLiteral() { exitRule(); }); try { - setState(1870); + setState(1874); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::FLOATING_LITERAL: { enterOuterAlt(_localctx, 1); - setState(1862); + setState(1866); match(ClickHouseParser::FLOATING_LITERAL); break; } case ClickHouseParser::DOT: { enterOuterAlt(_localctx, 2); - setState(1863); + setState(1867); match(ClickHouseParser::DOT); - setState(1864); + setState(1868); _la = _input->LA(1); if (!(_la == ClickHouseParser::OCTAL_LITERAL @@ -16743,16 +16619,16 @@ ClickHouseParser::FloatingLiteralContext* ClickHouseParser::floatingLiteral() { case ClickHouseParser::DECIMAL_LITERAL: { enterOuterAlt(_localctx, 3); - setState(1865); + setState(1869); match(ClickHouseParser::DECIMAL_LITERAL); - setState(1866); + setState(1870); match(ClickHouseParser::DOT); - setState(1868); + setState(1872); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 248, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 249, _ctx)) { case 1: { - setState(1867); + setState(1871); _la = _input->LA(1); if (!(_la == ClickHouseParser::OCTAL_LITERAL @@ -16827,7 +16703,6 @@ size_t ClickHouseParser::NumberLiteralContext::getRuleIndex() const { return ClickHouseParser::RuleNumberLiteral; } - antlrcpp::Any ClickHouseParser::NumberLiteralContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitNumberLiteral(this); @@ -16845,14 +16720,14 @@ ClickHouseParser::NumberLiteralContext* ClickHouseParser::numberLiteral() { }); try { enterOuterAlt(_localctx, 1); - setState(1873); + setState(1877); _errHandler->sync(this); _la = _input->LA(1); if (_la == ClickHouseParser::DASH || _la == ClickHouseParser::PLUS) { - setState(1872); + setState(1876); _la = _input->LA(1); if (!(_la == ClickHouseParser::DASH @@ -16864,41 +16739,41 @@ ClickHouseParser::NumberLiteralContext* ClickHouseParser::numberLiteral() { consume(); } } - setState(1881); + setState(1885); _errHandler->sync(this); - switch (getInterpreter()->adaptivePredict(_input, 251, _ctx)) { + switch (getInterpreter()->adaptivePredict(_input, 252, _ctx)) { case 1: { - setState(1875); + setState(1879); floatingLiteral(); break; } case 2: { - setState(1876); + setState(1880); match(ClickHouseParser::OCTAL_LITERAL); break; } case 3: { - setState(1877); + setState(1881); match(ClickHouseParser::DECIMAL_LITERAL); break; } case 4: { - setState(1878); + setState(1882); match(ClickHouseParser::HEXADECIMAL_LITERAL); break; } case 5: { - setState(1879); + setState(1883); match(ClickHouseParser::INF); break; } case 6: { - setState(1880); + setState(1884); match(ClickHouseParser::NAN_SQL); break; } @@ -16938,7 +16813,6 @@ size_t ClickHouseParser::LiteralContext::getRuleIndex() const { return ClickHouseParser::RuleLiteral; } - antlrcpp::Any ClickHouseParser::LiteralContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitLiteral(this); @@ -16954,7 +16828,7 @@ ClickHouseParser::LiteralContext* ClickHouseParser::literal() { exitRule(); }); try { - setState(1886); + setState(1890); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::INF: @@ -16967,21 +16841,21 @@ ClickHouseParser::LiteralContext* ClickHouseParser::literal() { case ClickHouseParser::DOT: case ClickHouseParser::PLUS: { enterOuterAlt(_localctx, 1); - setState(1883); + setState(1887); numberLiteral(); break; } case ClickHouseParser::STRING_LITERAL: { enterOuterAlt(_localctx, 2); - setState(1884); + setState(1888); match(ClickHouseParser::STRING_LITERAL); break; } case ClickHouseParser::NULL_SQL: { enterOuterAlt(_localctx, 3); - setState(1885); + setState(1889); match(ClickHouseParser::NULL_SQL); break; } @@ -17043,7 +16917,6 @@ size_t ClickHouseParser::IntervalContext::getRuleIndex() const { return ClickHouseParser::RuleInterval; } - antlrcpp::Any ClickHouseParser::IntervalContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitInterval(this); @@ -17061,16 +16934,16 @@ ClickHouseParser::IntervalContext* ClickHouseParser::interval() { }); try { enterOuterAlt(_localctx, 1); - setState(1888); + setState(1892); _la = _input->LA(1); - if (!(_la == ClickHouseParser::DAY || ((((_la - 72) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 72)) & ((1ULL << (ClickHouseParser::HOUR - 72)) - | (1ULL << (ClickHouseParser::MINUTE - 72)) - | (1ULL << (ClickHouseParser::MONTH - 72)) - | (1ULL << (ClickHouseParser::QUARTER - 72)))) != 0) || ((((_la - 137) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 137)) & ((1ULL << (ClickHouseParser::SECOND - 137)) - | (1ULL << (ClickHouseParser::WEEK - 137)) - | (1ULL << (ClickHouseParser::YEAR - 137)))) != 0))) { + if (!(_la == ClickHouseParser::DAY || ((((_la - 73) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 73)) & ((1ULL << (ClickHouseParser::HOUR - 73)) + | (1ULL << (ClickHouseParser::MINUTE - 73)) + | (1ULL << (ClickHouseParser::MONTH - 73)) + | (1ULL << (ClickHouseParser::QUARTER - 73)))) != 0) || ((((_la - 138) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 138)) & ((1ULL << (ClickHouseParser::SECOND - 138)) + | (1ULL << (ClickHouseParser::WEEK - 138)) + | (1ULL << (ClickHouseParser::YEAR - 138)))) != 0))) { _errHandler->recoverInline(this); } else { @@ -17138,6 +17011,10 @@ tree::TerminalNode* ClickHouseParser::KeywordContext::ASOF() { return getToken(ClickHouseParser::ASOF, 0); } +tree::TerminalNode* ClickHouseParser::KeywordContext::AST() { + return getToken(ClickHouseParser::AST, 0); +} + tree::TerminalNode* ClickHouseParser::KeywordContext::ASYNC() { return getToken(ClickHouseParser::ASYNC, 0); } @@ -17775,7 +17652,6 @@ size_t ClickHouseParser::KeywordContext::getRuleIndex() const { return ClickHouseParser::RuleKeyword; } - antlrcpp::Any ClickHouseParser::KeywordContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitKeyword(this); @@ -17793,7 +17669,7 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { }); try { enterOuterAlt(_localctx, 1); - setState(1890); + setState(1894); _la = _input->LA(1); if (!((((_la & ~ 0x3fULL) == 0) && ((1ULL << _la) & ((1ULL << ClickHouseParser::AFTER) @@ -17807,6 +17683,7 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { | (1ULL << ClickHouseParser::AS) | (1ULL << ClickHouseParser::ASCENDING) | (1ULL << ClickHouseParser::ASOF) + | (1ULL << ClickHouseParser::AST) | (1ULL << ClickHouseParser::ASYNC) | (1ULL << ClickHouseParser::ATTACH) | (1ULL << ClickHouseParser::BETWEEN) @@ -17855,9 +17732,9 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { | (1ULL << ClickHouseParser::FIRST) | (1ULL << ClickHouseParser::FLUSH) | (1ULL << ClickHouseParser::FOR) - | (1ULL << ClickHouseParser::FORMAT) - | (1ULL << ClickHouseParser::FREEZE))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FROM - 64)) + | (1ULL << ClickHouseParser::FORMAT))) != 0) || ((((_la - 64) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 64)) & ((1ULL << (ClickHouseParser::FREEZE - 64)) + | (1ULL << (ClickHouseParser::FROM - 64)) | (1ULL << (ClickHouseParser::FULL - 64)) | (1ULL << (ClickHouseParser::FUNCTION - 64)) | (1ULL << (ClickHouseParser::GLOBAL - 64)) @@ -17890,8 +17767,8 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { | (1ULL << (ClickHouseParser::LIVE - 64)) | (1ULL << (ClickHouseParser::LOCAL - 64)) | (1ULL << (ClickHouseParser::LOGS - 64)) - | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MATERIALIZE - 64)) + | (1ULL << (ClickHouseParser::MATERIALIZED - 64)) | (1ULL << (ClickHouseParser::MAX - 64)) | (1ULL << (ClickHouseParser::MERGES - 64)) | (1ULL << (ClickHouseParser::MIN - 64)) @@ -17911,9 +17788,9 @@ ClickHouseParser::KeywordContext* ClickHouseParser::keyword() { | (1ULL << (ClickHouseParser::PARTITION - 64)) | (1ULL << (ClickHouseParser::POPULATE - 64)) | (1ULL << (ClickHouseParser::PREWHERE - 64)) - | (1ULL << (ClickHouseParser::PRIMARY - 64)) - | (1ULL << (ClickHouseParser::RANGE - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RELOAD - 128)) + | (1ULL << (ClickHouseParser::PRIMARY - 64)))) != 0) || ((((_la - 128) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 128)) & ((1ULL << (ClickHouseParser::RANGE - 128)) + | (1ULL << (ClickHouseParser::RELOAD - 128)) | (1ULL << (ClickHouseParser::REMOVE - 128)) | (1ULL << (ClickHouseParser::RENAME - 128)) | (1ULL << (ClickHouseParser::REPLACE - 128)) @@ -18009,7 +17886,6 @@ size_t ClickHouseParser::KeywordForAliasContext::getRuleIndex() const { return ClickHouseParser::RuleKeywordForAlias; } - antlrcpp::Any ClickHouseParser::KeywordForAliasContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitKeywordForAlias(this); @@ -18027,13 +17903,13 @@ ClickHouseParser::KeywordForAliasContext* ClickHouseParser::keywordForAlias() { }); try { enterOuterAlt(_localctx, 1); - setState(1892); + setState(1896); _la = _input->LA(1); - if (!(((((_la - 33) & ~ 0x3fULL) == 0) && - ((1ULL << (_la - 33)) & ((1ULL << (ClickHouseParser::DATE - 33)) - | (1ULL << (ClickHouseParser::FIRST - 33)) - | (1ULL << (ClickHouseParser::ID - 33)) - | (1ULL << (ClickHouseParser::KEY - 33)))) != 0))) { + if (!(((((_la - 34) & ~ 0x3fULL) == 0) && + ((1ULL << (_la - 34)) & ((1ULL << (ClickHouseParser::DATE - 34)) + | (1ULL << (ClickHouseParser::FIRST - 34)) + | (1ULL << (ClickHouseParser::ID - 34)) + | (1ULL << (ClickHouseParser::KEY - 34)))) != 0))) { _errHandler->recoverInline(this); } else { @@ -18070,7 +17946,6 @@ size_t ClickHouseParser::AliasContext::getRuleIndex() const { return ClickHouseParser::RuleAlias; } - antlrcpp::Any ClickHouseParser::AliasContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitAlias(this); @@ -18086,12 +17961,12 @@ ClickHouseParser::AliasContext* ClickHouseParser::alias() { exitRule(); }); try { - setState(1896); + setState(1900); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1894); + setState(1898); match(ClickHouseParser::IDENTIFIER); break; } @@ -18101,7 +17976,7 @@ ClickHouseParser::AliasContext* ClickHouseParser::alias() { case ClickHouseParser::ID: case ClickHouseParser::KEY: { enterOuterAlt(_localctx, 2); - setState(1895); + setState(1899); keywordForAlias(); break; } @@ -18143,7 +18018,6 @@ size_t ClickHouseParser::IdentifierContext::getRuleIndex() const { return ClickHouseParser::RuleIdentifier; } - antlrcpp::Any ClickHouseParser::IdentifierContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitIdentifier(this); @@ -18159,12 +18033,12 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { exitRule(); }); try { - setState(1901); + setState(1905); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1898); + setState(1902); match(ClickHouseParser::IDENTIFIER); break; } @@ -18178,7 +18052,7 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::WEEK: case ClickHouseParser::YEAR: { enterOuterAlt(_localctx, 2); - setState(1899); + setState(1903); interval(); break; } @@ -18194,6 +18068,7 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::AS: case ClickHouseParser::ASCENDING: case ClickHouseParser::ASOF: + case ClickHouseParser::AST: case ClickHouseParser::ASYNC: case ClickHouseParser::ATTACH: case ClickHouseParser::BETWEEN: @@ -18277,8 +18152,8 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::LIVE: case ClickHouseParser::LOCAL: case ClickHouseParser::LOGS: - case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MATERIALIZE: + case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MAX: case ClickHouseParser::MERGES: case ClickHouseParser::MIN: @@ -18353,7 +18228,7 @@ ClickHouseParser::IdentifierContext* ClickHouseParser::identifier() { case ClickHouseParser::JSON_FALSE: case ClickHouseParser::JSON_TRUE: { enterOuterAlt(_localctx, 3); - setState(1900); + setState(1904); keyword(); break; } @@ -18391,7 +18266,6 @@ size_t ClickHouseParser::IdentifierOrNullContext::getRuleIndex() const { return ClickHouseParser::RuleIdentifierOrNull; } - antlrcpp::Any ClickHouseParser::IdentifierOrNullContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitIdentifierOrNull(this); @@ -18407,7 +18281,7 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() exitRule(); }); try { - setState(1905); + setState(1909); _errHandler->sync(this); switch (_input->LA(1)) { case ClickHouseParser::AFTER: @@ -18421,6 +18295,7 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() case ClickHouseParser::AS: case ClickHouseParser::ASCENDING: case ClickHouseParser::ASOF: + case ClickHouseParser::AST: case ClickHouseParser::ASYNC: case ClickHouseParser::ATTACH: case ClickHouseParser::BETWEEN: @@ -18506,8 +18381,8 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() case ClickHouseParser::LIVE: case ClickHouseParser::LOCAL: case ClickHouseParser::LOGS: - case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MATERIALIZE: + case ClickHouseParser::MATERIALIZED: case ClickHouseParser::MAX: case ClickHouseParser::MERGES: case ClickHouseParser::MIN: @@ -18589,14 +18464,14 @@ ClickHouseParser::IdentifierOrNullContext* ClickHouseParser::identifierOrNull() case ClickHouseParser::JSON_TRUE: case ClickHouseParser::IDENTIFIER: { enterOuterAlt(_localctx, 1); - setState(1903); + setState(1907); identifier(); break; } case ClickHouseParser::NULL_SQL: { enterOuterAlt(_localctx, 2); - setState(1904); + setState(1908); match(ClickHouseParser::NULL_SQL); break; } @@ -18638,7 +18513,6 @@ size_t ClickHouseParser::EnumValueContext::getRuleIndex() const { return ClickHouseParser::RuleEnumValue; } - antlrcpp::Any ClickHouseParser::EnumValueContext::accept(tree::ParseTreeVisitor *visitor) { if (auto parserVisitor = dynamic_cast(visitor)) return parserVisitor->visitEnumValue(this); @@ -18655,11 +18529,11 @@ ClickHouseParser::EnumValueContext* ClickHouseParser::enumValue() { }); try { enterOuterAlt(_localctx, 1); - setState(1907); + setState(1911); match(ClickHouseParser::STRING_LITERAL); - setState(1908); + setState(1912); match(ClickHouseParser::EQ_SINGLE); - setState(1909); + setState(1913); numberLiteral(); } @@ -18817,7 +18691,7 @@ std::vector ClickHouseParser::_literalNames = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", - "", "'false'", "'true'", "", "", "", "", "", "", "'->'", "'*'", "'`'", + "", "", "'false'", "'true'", "", "", "", "", "", "", "'->'", "'*'", "'`'", "'\\'", "':'", "','", "'||'", "'-'", "'.'", "'=='", "'='", "'>='", "'>'", "'{'", "'['", "'<='", "'('", "'<'", "", "'%'", "'+'", "'?'", "'\"'", "'''", "'}'", "']'", "')'", "';'", "'/'", "'_'" @@ -18825,35 +18699,36 @@ std::vector ClickHouseParser::_literalNames = { std::vector ClickHouseParser::_symbolicNames = { "", "ADD", "AFTER", "ALIAS", "ALL", "ALTER", "AND", "ANTI", "ANY", "ARRAY", - "AS", "ASCENDING", "ASOF", "ASYNC", "ATTACH", "BETWEEN", "BOTH", "BY", - "CASE", "CAST", "CHECK", "CLEAR", "CLUSTER", "CODEC", "COLLATE", "COLUMN", - "COMMENT", "CONSTRAINT", "CREATE", "CROSS", "CUBE", "DATABASE", "DATABASES", - "DATE", "DAY", "DEDUPLICATE", "DEFAULT", "DELAY", "DELETE", "DESC", "DESCENDING", - "DESCRIBE", "DETACH", "DICTIONARIES", "DICTIONARY", "DISK", "DISTINCT", - "DISTRIBUTED", "DROP", "ELSE", "END", "ENGINE", "EVENTS", "EXISTS", "EXPLAIN", - "EXPRESSION", "EXTRACT", "FETCHES", "FINAL", "FIRST", "FLUSH", "FOR", - "FORMAT", "FREEZE", "FROM", "FULL", "FUNCTION", "GLOBAL", "GRANULARITY", - "GROUP", "HAVING", "HIERARCHICAL", "HOUR", "ID", "IF", "ILIKE", "IN", - "INDEX", "INF", "INJECTIVE", "INNER", "INSERT", "INTERVAL", "INTO", "IS", - "IS_OBJECT_ID", "JOIN", "KEY", "KILL", "LAST", "LAYOUT", "LEADING", "LEFT", - "LIFETIME", "LIKE", "LIMIT", "LIVE", "LOCAL", "LOGS", "MATERIALIZED", - "MATERIALIZE", "MAX", "MERGES", "MIN", "MINUTE", "MODIFY", "MONTH", "MOVE", - "MUTATION", "NAN_SQL", "NO", "NOT", "NULL_SQL", "NULLS", "OFFSET", "ON", - "OPTIMIZE", "OR", "ORDER", "OUTER", "OUTFILE", "PARTITION", "POPULATE", - "PREWHERE", "PRIMARY", "PROJECTION", "QUARTER", "RANGE", "RELOAD", "REMOVE", - "RENAME", "REPLACE", "REPLICA", "REPLICATED", "RIGHT", "ROLLUP", "SAMPLE", - "SECOND", "SELECT", "SEMI", "SENDS", "SET", "SETTINGS", "SHOW", "SOURCE", - "START", "STOP", "SUBSTRING", "SYNC", "SYNTAX", "SYSTEM", "TABLE", "TABLES", - "TEMPORARY", "TEST", "THEN", "TIES", "TIMEOUT", "TIMESTAMP", "TO", "TOP", - "TOTALS", "TRAILING", "TRIM", "TRUNCATE", "TTL", "TYPE", "UNION", "UPDATE", - "USE", "USING", "UUID", "VALUES", "VIEW", "VOLUME", "WATCH", "WEEK", "WHEN", - "WHERE", "WITH", "YEAR", "JSON_FALSE", "JSON_TRUE", "IDENTIFIER", "FLOATING_LITERAL", - "OCTAL_LITERAL", "DECIMAL_LITERAL", "HEXADECIMAL_LITERAL", "STRING_LITERAL", - "ARROW", "ASTERISK", "BACKQUOTE", "BACKSLASH", "COLON", "COMMA", "CONCAT", - "DASH", "DOT", "EQ_DOUBLE", "EQ_SINGLE", "GE", "GT", "LBRACE", "LBRACKET", - "LE", "LPAREN", "LT", "NOT_EQ", "PERCENT", "PLUS", "QUERY", "QUOTE_DOUBLE", - "QUOTE_SINGLE", "RBRACE", "RBRACKET", "RPAREN", "SEMICOLON", "SLASH", - "UNDERSCORE", "MULTI_LINE_COMMENT", "SINGLE_LINE_COMMENT", "WHITESPACE" + "AS", "ASCENDING", "ASOF", "AST", "ASYNC", "ATTACH", "BETWEEN", "BOTH", + "BY", "CASE", "CAST", "CHECK", "CLEAR", "CLUSTER", "CODEC", "COLLATE", + "COLUMN", "COMMENT", "CONSTRAINT", "CREATE", "CROSS", "CUBE", "DATABASE", + "DATABASES", "DATE", "DAY", "DEDUPLICATE", "DEFAULT", "DELAY", "DELETE", + "DESC", "DESCENDING", "DESCRIBE", "DETACH", "DICTIONARIES", "DICTIONARY", + "DISK", "DISTINCT", "DISTRIBUTED", "DROP", "ELSE", "END", "ENGINE", "EVENTS", + "EXISTS", "EXPLAIN", "EXPRESSION", "EXTRACT", "FETCHES", "FINAL", "FIRST", + "FLUSH", "FOR", "FORMAT", "FREEZE", "FROM", "FULL", "FUNCTION", "GLOBAL", + "GRANULARITY", "GROUP", "HAVING", "HIERARCHICAL", "HOUR", "ID", "IF", + "ILIKE", "IN", "INDEX", "INF", "INJECTIVE", "INNER", "INSERT", "INTERVAL", + "INTO", "IS", "IS_OBJECT_ID", "JOIN", "KEY", "KILL", "LAST", "LAYOUT", + "LEADING", "LEFT", "LIFETIME", "LIKE", "LIMIT", "LIVE", "LOCAL", "LOGS", + "MATERIALIZE", "MATERIALIZED", "MAX", "MERGES", "MIN", "MINUTE", "MODIFY", + "MONTH", "MOVE", "MUTATION", "NAN_SQL", "NO", "NOT", "NULL_SQL", "NULLS", + "OFFSET", "ON", "OPTIMIZE", "OR", "ORDER", "OUTER", "OUTFILE", "PARTITION", + "POPULATE", "PREWHERE", "PRIMARY", "PROJECTION", "QUARTER", "RANGE", "RELOAD", + "REMOVE", "RENAME", "REPLACE", "REPLICA", "REPLICATED", "RIGHT", "ROLLUP", + "SAMPLE", "SECOND", "SELECT", "SEMI", "SENDS", "SET", "SETTINGS", "SHOW", + "SOURCE", "START", "STOP", "SUBSTRING", "SYNC", "SYNTAX", "SYSTEM", "TABLE", + "TABLES", "TEMPORARY", "TEST", "THEN", "TIES", "TIMEOUT", "TIMESTAMP", + "TO", "TOP", "TOTALS", "TRAILING", "TRIM", "TRUNCATE", "TTL", "TYPE", + "UNION", "UPDATE", "USE", "USING", "UUID", "VALUES", "VIEW", "VOLUME", + "WATCH", "WEEK", "WHEN", "WHERE", "WITH", "YEAR", "JSON_FALSE", "JSON_TRUE", + "IDENTIFIER", "FLOATING_LITERAL", "OCTAL_LITERAL", "DECIMAL_LITERAL", + "HEXADECIMAL_LITERAL", "STRING_LITERAL", "ARROW", "ASTERISK", "BACKQUOTE", + "BACKSLASH", "COLON", "COMMA", "CONCAT", "DASH", "DOT", "EQ_DOUBLE", "EQ_SINGLE", + "GE", "GT", "LBRACE", "LBRACKET", "LE", "LPAREN", "LT", "NOT_EQ", "PERCENT", + "PLUS", "QUERY", "QUOTE_DOUBLE", "QUOTE_SINGLE", "RBRACE", "RBRACKET", + "RPAREN", "SEMICOLON", "SLASH", "UNDERSCORE", "MULTI_LINE_COMMENT", "SINGLE_LINE_COMMENT", + "WHITESPACE" }; dfa::Vocabulary ClickHouseParser::_vocabulary(_literalNames, _symbolicNames); @@ -18876,7 +18751,7 @@ ClickHouseParser::Initializer::Initializer() { _serializedATN = { 0x3, 0x608b, 0xa72a, 0x8133, 0xb9ed, 0x417c, 0x3be7, 0x7786, 0x5964, - 0x3, 0xdf, 0x77a, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, 0x4, 0x4, + 0x3, 0xe0, 0x77e, 0x4, 0x2, 0x9, 0x2, 0x4, 0x3, 0x9, 0x3, 0x4, 0x4, 0x9, 0x4, 0x4, 0x5, 0x9, 0x5, 0x4, 0x6, 0x9, 0x6, 0x4, 0x7, 0x9, 0x7, 0x4, 0x8, 0x9, 0x8, 0x4, 0x9, 0x9, 0x9, 0x4, 0xa, 0x9, 0xa, 0x4, 0xb, 0x9, 0xb, 0x4, 0xc, 0x9, 0xc, 0x4, 0xd, 0x9, 0xd, 0x4, 0xe, 0x9, 0xe, @@ -19041,389 +18916,390 @@ ClickHouseParser::Initializer::Initializer() { 0x3fc, 0xa, 0x2b, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x5, 0x2c, 0x404, 0xa, 0x2c, 0x3, 0x2c, 0x3, 0x2c, 0x5, 0x2c, 0x408, 0xa, 0x2c, 0x3, 0x2c, 0x5, 0x2c, 0x40b, 0xa, 0x2c, 0x3, 0x2d, - 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, - 0x2e, 0x414, 0xa, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, - 0x419, 0xa, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x41c, 0xa, 0x2e, 0x3, 0x2e, - 0x3, 0x2e, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x7, 0x2f, 0x424, - 0xa, 0x2f, 0xc, 0x2f, 0xe, 0x2f, 0x427, 0xb, 0x2f, 0x3, 0x2f, 0x3, 0x2f, - 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, 0x5, 0x30, 0x430, - 0xa, 0x30, 0x3, 0x30, 0x3, 0x30, 0x5, 0x30, 0x434, 0xa, 0x30, 0x3, 0x31, - 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, 0x439, 0xa, 0x31, 0x3, 0x31, 0x3, 0x31, - 0x5, 0x31, 0x43d, 0xa, 0x31, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, - 0x5, 0x32, 0x443, 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x446, 0xa, 0x32, - 0x3, 0x32, 0x5, 0x32, 0x449, 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x44c, - 0xa, 0x32, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, - 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x7, 0x33, 0x458, - 0xa, 0x33, 0xc, 0x33, 0xe, 0x33, 0x45b, 0xb, 0x33, 0x3, 0x33, 0x5, 0x33, - 0x45e, 0xa, 0x33, 0x3, 0x34, 0x3, 0x34, 0x5, 0x34, 0x462, 0xa, 0x34, - 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, 0x5, 0x34, 0x467, 0xa, 0x34, 0x3, 0x34, - 0x5, 0x34, 0x46a, 0xa, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x35, 0x3, 0x35, - 0x3, 0x35, 0x3, 0x35, 0x7, 0x35, 0x472, 0xa, 0x35, 0xc, 0x35, 0xe, 0x35, - 0x475, 0xb, 0x35, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, - 0x5, 0x36, 0x47c, 0xa, 0x36, 0x3, 0x37, 0x5, 0x37, 0x47f, 0xa, 0x37, - 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x483, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, - 0x486, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x48a, 0xa, 0x37, - 0x3, 0x37, 0x5, 0x37, 0x48d, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x490, - 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x493, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, - 0x496, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49a, 0xa, 0x37, - 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49e, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, - 0x4a1, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4a4, 0xa, 0x37, 0x3, 0x37, - 0x5, 0x37, 0x4a7, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4aa, 0xa, 0x37, - 0x3, 0x37, 0x5, 0x37, 0x4ad, 0xa, 0x37, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, - 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x3, 0x39, 0x5, 0x39, 0x4b6, 0xa, 0x39, - 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3a, 0x3, 0x3b, 0x5, 0x3b, 0x4bc, 0xa, 0x3b, - 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3c, 0x3, 0x3c, 0x3, - 0x3c, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3d, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, - 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x5, 0x3e, 0x4d0, - 0xa, 0x3e, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x40, 0x3, 0x40, 0x3, - 0x40, 0x3, 0x40, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x42, - 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, 0x3, 0x43, 0x3, 0x43, 0x3, - 0x43, 0x3, 0x43, 0x5, 0x43, 0x4e6, 0xa, 0x43, 0x3, 0x44, 0x3, 0x44, - 0x3, 0x44, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, 0x4ee, 0xa, 0x45, - 0x3, 0x45, 0x5, 0x45, 0x4f1, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, - 0x3, 0x45, 0x5, 0x45, 0x4f7, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, - 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, 0x4ff, 0xa, 0x45, 0x3, 0x45, - 0x5, 0x45, 0x502, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, - 0x7, 0x45, 0x508, 0xa, 0x45, 0xc, 0x45, 0xe, 0x45, 0x50b, 0xb, 0x45, - 0x3, 0x46, 0x5, 0x46, 0x50e, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, - 0x5, 0x46, 0x513, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x516, 0xa, 0x46, - 0x3, 0x46, 0x5, 0x46, 0x519, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x3, 0x2d, 0x5, 0x2d, 0x413, + 0xa, 0x2d, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x418, 0xa, 0x2e, + 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x5, 0x2e, 0x41d, 0xa, 0x2e, 0x3, 0x2e, + 0x5, 0x2e, 0x420, 0xa, 0x2e, 0x3, 0x2e, 0x3, 0x2e, 0x3, 0x2f, 0x3, 0x2f, + 0x3, 0x2f, 0x3, 0x2f, 0x7, 0x2f, 0x428, 0xa, 0x2f, 0xc, 0x2f, 0xe, 0x2f, + 0x42b, 0xb, 0x2f, 0x3, 0x2f, 0x3, 0x2f, 0x3, 0x30, 0x3, 0x30, 0x3, 0x30, + 0x3, 0x30, 0x3, 0x30, 0x5, 0x30, 0x434, 0xa, 0x30, 0x3, 0x30, 0x3, 0x30, + 0x5, 0x30, 0x438, 0xa, 0x30, 0x3, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, + 0x43d, 0xa, 0x31, 0x3, 0x31, 0x3, 0x31, 0x5, 0x31, 0x441, 0xa, 0x31, + 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x3, 0x32, 0x5, 0x32, 0x447, 0xa, 0x32, + 0x3, 0x32, 0x5, 0x32, 0x44a, 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x44d, + 0xa, 0x32, 0x3, 0x32, 0x5, 0x32, 0x450, 0xa, 0x32, 0x3, 0x33, 0x3, 0x33, + 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, 0x33, 0x3, + 0x33, 0x3, 0x33, 0x7, 0x33, 0x45c, 0xa, 0x33, 0xc, 0x33, 0xe, 0x33, + 0x45f, 0xb, 0x33, 0x3, 0x33, 0x5, 0x33, 0x462, 0xa, 0x33, 0x3, 0x34, + 0x3, 0x34, 0x5, 0x34, 0x466, 0xa, 0x34, 0x3, 0x34, 0x3, 0x34, 0x3, 0x34, + 0x5, 0x34, 0x46b, 0xa, 0x34, 0x3, 0x34, 0x5, 0x34, 0x46e, 0xa, 0x34, + 0x3, 0x34, 0x3, 0x34, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x3, 0x35, 0x7, + 0x35, 0x476, 0xa, 0x35, 0xc, 0x35, 0xe, 0x35, 0x479, 0xb, 0x35, 0x3, + 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x3, 0x36, 0x5, 0x36, 0x480, + 0xa, 0x36, 0x3, 0x37, 0x5, 0x37, 0x483, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, + 0x5, 0x37, 0x487, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x48a, 0xa, 0x37, + 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x48e, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, + 0x491, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x494, 0xa, 0x37, 0x3, 0x37, + 0x5, 0x37, 0x497, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49a, 0xa, 0x37, + 0x3, 0x37, 0x3, 0x37, 0x5, 0x37, 0x49e, 0xa, 0x37, 0x3, 0x37, 0x3, 0x37, + 0x5, 0x37, 0x4a2, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4a5, 0xa, 0x37, + 0x3, 0x37, 0x5, 0x37, 0x4a8, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4ab, + 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, 0x4ae, 0xa, 0x37, 0x3, 0x37, 0x5, 0x37, + 0x4b1, 0xa, 0x37, 0x3, 0x38, 0x3, 0x38, 0x3, 0x38, 0x3, 0x39, 0x3, 0x39, + 0x3, 0x39, 0x3, 0x39, 0x5, 0x39, 0x4ba, 0xa, 0x39, 0x3, 0x3a, 0x3, 0x3a, + 0x3, 0x3a, 0x3, 0x3b, 0x5, 0x3b, 0x4c0, 0xa, 0x3b, 0x3, 0x3b, 0x3, 0x3b, + 0x3, 0x3b, 0x3, 0x3b, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3c, 0x3, 0x3d, 0x3, + 0x3d, 0x3, 0x3d, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, + 0x3, 0x3e, 0x3, 0x3e, 0x3, 0x3e, 0x5, 0x3e, 0x4d4, 0xa, 0x3e, 0x3, 0x3f, + 0x3, 0x3f, 0x3, 0x3f, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, 0x40, 0x3, + 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x41, 0x3, 0x42, 0x3, 0x42, 0x3, 0x42, + 0x3, 0x42, 0x3, 0x42, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x3, 0x43, 0x5, + 0x43, 0x4ea, 0xa, 0x43, 0x3, 0x44, 0x3, 0x44, 0x3, 0x44, 0x3, 0x45, + 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, 0x4f2, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, + 0x4f5, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x5, 0x45, + 0x4fb, 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, + 0x3, 0x45, 0x5, 0x45, 0x503, 0xa, 0x45, 0x3, 0x45, 0x5, 0x45, 0x506, + 0xa, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x3, 0x45, 0x7, 0x45, 0x50c, + 0xa, 0x45, 0xc, 0x45, 0xe, 0x45, 0x50f, 0xb, 0x45, 0x3, 0x46, 0x5, 0x46, + 0x512, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x517, + 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x51a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x51d, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x521, 0xa, 0x46, - 0x3, 0x46, 0x5, 0x46, 0x524, 0xa, 0x46, 0x5, 0x46, 0x526, 0xa, 0x46, - 0x3, 0x46, 0x5, 0x46, 0x529, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x525, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x528, 0xa, 0x46, 0x5, 0x46, 0x52a, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, 0x52d, 0xa, 0x46, 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x531, 0xa, 0x46, - 0x3, 0x46, 0x5, 0x46, 0x534, 0xa, 0x46, 0x5, 0x46, 0x536, 0xa, 0x46, - 0x5, 0x46, 0x538, 0xa, 0x46, 0x3, 0x47, 0x5, 0x47, 0x53b, 0xa, 0x47, - 0x3, 0x47, 0x3, 0x47, 0x3, 0x47, 0x5, 0x47, 0x540, 0xa, 0x47, 0x3, 0x48, - 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, - 0x48, 0x3, 0x48, 0x5, 0x48, 0x54b, 0xa, 0x48, 0x3, 0x49, 0x3, 0x49, - 0x3, 0x49, 0x3, 0x49, 0x5, 0x49, 0x551, 0xa, 0x49, 0x3, 0x4a, 0x3, 0x4a, - 0x3, 0x4a, 0x5, 0x4a, 0x556, 0xa, 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4b, - 0x7, 0x4b, 0x55b, 0xa, 0x4b, 0xc, 0x4b, 0xe, 0x4b, 0x55e, 0xb, 0x4b, - 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x562, 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, + 0x3, 0x46, 0x3, 0x46, 0x5, 0x46, 0x535, 0xa, 0x46, 0x3, 0x46, 0x5, 0x46, + 0x538, 0xa, 0x46, 0x5, 0x46, 0x53a, 0xa, 0x46, 0x5, 0x46, 0x53c, 0xa, + 0x46, 0x3, 0x47, 0x5, 0x47, 0x53f, 0xa, 0x47, 0x3, 0x47, 0x3, 0x47, + 0x3, 0x47, 0x5, 0x47, 0x544, 0xa, 0x47, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, + 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x3, 0x48, 0x5, + 0x48, 0x54f, 0xa, 0x48, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, 0x3, 0x49, + 0x5, 0x49, 0x555, 0xa, 0x49, 0x3, 0x4a, 0x3, 0x4a, 0x3, 0x4a, 0x5, 0x4a, + 0x55a, 0xa, 0x4a, 0x3, 0x4b, 0x3, 0x4b, 0x3, 0x4b, 0x7, 0x4b, 0x55f, + 0xa, 0x4b, 0xc, 0x4b, 0xe, 0x4b, 0x562, 0xb, 0x4b, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x566, 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x56a, - 0xa, 0x4c, 0x3, 0x4d, 0x3, 0x4d, 0x3, 0x4d, 0x5, 0x4d, 0x56f, 0xa, 0x4d, - 0x3, 0x4e, 0x3, 0x4e, 0x3, 0x4e, 0x7, 0x4e, 0x574, 0xa, 0x4e, 0xc, 0x4e, - 0xe, 0x4e, 0x577, 0xb, 0x4e, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, - 0x3, 0x50, 0x3, 0x50, 0x3, 0x50, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, + 0xa, 0x4c, 0x3, 0x4c, 0x3, 0x4c, 0x5, 0x4c, 0x56e, 0xa, 0x4c, 0x3, 0x4d, + 0x3, 0x4d, 0x3, 0x4d, 0x5, 0x4d, 0x573, 0xa, 0x4d, 0x3, 0x4e, 0x3, 0x4e, + 0x3, 0x4e, 0x7, 0x4e, 0x578, 0xa, 0x4e, 0xc, 0x4e, 0xe, 0x4e, 0x57b, + 0xb, 0x4e, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x4f, 0x3, 0x50, 0x3, + 0x50, 0x3, 0x50, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, + 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, + 0x51, 0x58f, 0xa, 0x51, 0x3, 0x51, 0x5, 0x51, 0x592, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, - 0x3, 0x51, 0x5, 0x51, 0x58b, 0xa, 0x51, 0x3, 0x51, 0x5, 0x51, 0x58e, - 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, - 0x51, 0x3, 0x51, 0x5, 0x51, 0x597, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, - 0x5, 0x51, 0x59b, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, - 0x5a0, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a5, - 0xa, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a8, 0xa, 0x51, 0x5, 0x51, 0x5aa, - 0xa, 0x51, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, - 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, + 0x5, 0x51, 0x59b, 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x59f, + 0xa, 0x51, 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a4, 0xa, 0x51, + 0x3, 0x51, 0x3, 0x51, 0x3, 0x51, 0x5, 0x51, 0x5a9, 0xa, 0x51, 0x3, 0x51, + 0x5, 0x51, 0x5ac, 0xa, 0x51, 0x5, 0x51, 0x5ae, 0xa, 0x51, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, - 0x52, 0x3, 0x52, 0x5, 0x52, 0x5c0, 0xa, 0x52, 0x3, 0x52, 0x5, 0x52, - 0x5c3, 0xa, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, - 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5ce, 0xa, 0x52, - 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5d2, 0xa, 0x53, 0x3, 0x53, 0x5, 0x53, - 0x5d5, 0xa, 0x53, 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5d9, 0xa, 0x53, - 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5dd, 0xa, 0x53, 0x3, 0x54, 0x3, 0x54, - 0x3, 0x54, 0x3, 0x55, 0x3, 0x55, 0x3, 0x55, 0x5, 0x55, 0x5e5, 0xa, 0x55, - 0x3, 0x55, 0x3, 0x55, 0x5, 0x55, 0x5e9, 0xa, 0x55, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, - 0x56, 0x7, 0x56, 0x5f4, 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x5f7, 0xb, - 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x3, 0x56, 0x7, 0x56, 0x600, 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x603, - 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, - 0x56, 0x3, 0x56, 0x7, 0x56, 0x60c, 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, - 0x60f, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, - 0x5, 0x56, 0x616, 0xa, 0x56, 0x3, 0x56, 0x3, 0x56, 0x5, 0x56, 0x61a, - 0xa, 0x56, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, 0x7, 0x57, 0x61f, 0xa, 0x57, - 0xc, 0x57, 0xe, 0x57, 0x622, 0xb, 0x57, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, - 0x5, 0x58, 0x627, 0xa, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, - 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x62f, 0xa, 0x58, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x5, 0x59, 0x634, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x6, 0x59, 0x63b, 0xa, 0x59, 0xd, 0x59, 0xe, 0x59, - 0x63c, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x641, 0xa, 0x59, 0x3, 0x59, + 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, + 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x5, + 0x52, 0x5c4, 0xa, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5c7, 0xa, 0x52, 0x3, + 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, 0x3, 0x52, + 0x3, 0x52, 0x3, 0x52, 0x5, 0x52, 0x5d2, 0xa, 0x52, 0x3, 0x53, 0x3, 0x53, + 0x5, 0x53, 0x5d6, 0xa, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5d9, 0xa, 0x53, + 0x3, 0x53, 0x3, 0x53, 0x5, 0x53, 0x5dd, 0xa, 0x53, 0x3, 0x53, 0x3, 0x53, + 0x5, 0x53, 0x5e1, 0xa, 0x53, 0x3, 0x54, 0x3, 0x54, 0x3, 0x54, 0x3, 0x55, + 0x3, 0x55, 0x3, 0x55, 0x5, 0x55, 0x5e9, 0xa, 0x55, 0x3, 0x55, 0x3, 0x55, + 0x5, 0x55, 0x5ed, 0xa, 0x55, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x5f8, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x5fb, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x604, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x607, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x7, 0x56, 0x610, + 0xa, 0x56, 0xc, 0x56, 0xe, 0x56, 0x613, 0xb, 0x56, 0x3, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x3, 0x56, 0x3, 0x56, 0x5, 0x56, 0x61a, 0xa, 0x56, 0x3, 0x56, + 0x3, 0x56, 0x5, 0x56, 0x61e, 0xa, 0x56, 0x3, 0x57, 0x3, 0x57, 0x3, 0x57, + 0x7, 0x57, 0x623, 0xa, 0x57, 0xc, 0x57, 0xe, 0x57, 0x626, 0xb, 0x57, + 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x62b, 0xa, 0x58, 0x3, 0x58, + 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x3, 0x58, 0x5, 0x58, 0x633, + 0xa, 0x58, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x638, 0xa, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x6, 0x59, 0x63f, + 0xa, 0x59, 0xd, 0x59, 0xe, 0x59, 0x640, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, + 0x645, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x660, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x5, 0x59, 0x671, 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x674, 0xa, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x678, 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, - 0x67b, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x687, + 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x664, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x698, 0xa, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x5, 0x59, 0x69c, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x675, 0xa, 0x59, 0x3, 0x59, + 0x5, 0x59, 0x678, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x67c, + 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x67f, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, - 0x6ad, 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b0, 0xa, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x5, 0x59, 0x6b4, 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b7, - 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6c2, 0xa, 0x59, + 0x59, 0x3, 0x59, 0x5, 0x59, 0x68b, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x5, 0x59, 0x69c, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6a0, + 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, + 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b1, 0xa, 0x59, 0x3, 0x59, + 0x5, 0x59, 0x6b4, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6b8, + 0xa, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6bb, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, - 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6da, 0xa, 0x59, 0x3, 0x59, - 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, 0x59, 0x6e1, 0xa, 0x59, - 0x7, 0x59, 0x6e3, 0xa, 0x59, 0xc, 0x59, 0xe, 0x59, 0x6e6, 0xb, 0x59, - 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, 0x7, 0x5a, 0x6eb, 0xa, 0x5a, 0xc, 0x5a, - 0xe, 0x5a, 0x6ee, 0xb, 0x5a, 0x3, 0x5b, 0x3, 0x5b, 0x5, 0x5b, 0x6f2, - 0xa, 0x5b, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x7, 0x5c, 0x6f8, - 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, 0x6fb, 0xb, 0x5c, 0x3, 0x5c, 0x3, 0x5c, - 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x7, 0x5c, 0x702, 0xa, 0x5c, 0xc, 0x5c, - 0xe, 0x5c, 0x705, 0xb, 0x5c, 0x5, 0x5c, 0x707, 0xa, 0x5c, 0x3, 0x5c, - 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x5, 0x5d, 0x70f, - 0xa, 0x5d, 0x3, 0x5d, 0x3, 0x5d, 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x5, - 0x5e, 0x716, 0xa, 0x5e, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, - 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x5, 0x5f, 0x71f, 0xa, 0x5f, 0x3, 0x5f, - 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x5, 0x5f, 0x725, 0xa, 0x5f, 0x7, 0x5f, - 0x727, 0xa, 0x5f, 0xc, 0x5f, 0xe, 0x5f, 0x72a, 0xb, 0x5f, 0x3, 0x60, - 0x3, 0x60, 0x3, 0x60, 0x5, 0x60, 0x72f, 0xa, 0x60, 0x3, 0x60, 0x3, 0x60, - 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, 0x5, 0x61, 0x736, 0xa, 0x61, 0x3, 0x61, - 0x3, 0x61, 0x3, 0x62, 0x3, 0x62, 0x3, 0x62, 0x7, 0x62, 0x73d, 0xa, 0x62, - 0xc, 0x62, 0xe, 0x62, 0x740, 0xb, 0x62, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, - 0x5, 0x63, 0x745, 0xa, 0x63, 0x3, 0x64, 0x3, 0x64, 0x3, 0x65, 0x3, 0x65, - 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x5, 0x65, 0x74f, 0xa, 0x65, - 0x5, 0x65, 0x751, 0xa, 0x65, 0x3, 0x66, 0x5, 0x66, 0x754, 0xa, 0x66, - 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x5, - 0x66, 0x75c, 0xa, 0x66, 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x5, 0x67, - 0x761, 0xa, 0x67, 0x3, 0x68, 0x3, 0x68, 0x3, 0x69, 0x3, 0x69, 0x3, 0x6a, - 0x3, 0x6a, 0x3, 0x6b, 0x3, 0x6b, 0x5, 0x6b, 0x76b, 0xa, 0x6b, 0x3, 0x6c, - 0x3, 0x6c, 0x3, 0x6c, 0x5, 0x6c, 0x770, 0xa, 0x6c, 0x3, 0x6d, 0x3, 0x6d, - 0x5, 0x6d, 0x774, 0xa, 0x6d, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, - 0x3, 0x6e, 0x2, 0x5, 0x88, 0xb0, 0xbc, 0x6f, 0x2, 0x4, 0x6, 0x8, 0xa, - 0xc, 0xe, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22, - 0x24, 0x26, 0x28, 0x2a, 0x2c, 0x2e, 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, - 0x3c, 0x3e, 0x40, 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e, 0x50, 0x52, - 0x54, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x62, 0x64, 0x66, 0x68, 0x6a, - 0x6c, 0x6e, 0x70, 0x72, 0x74, 0x76, 0x78, 0x7a, 0x7c, 0x7e, 0x80, 0x82, - 0x84, 0x86, 0x88, 0x8a, 0x8c, 0x8e, 0x90, 0x92, 0x94, 0x96, 0x98, 0x9a, - 0x9c, 0x9e, 0xa0, 0xa2, 0xa4, 0xa6, 0xa8, 0xaa, 0xac, 0xae, 0xb0, 0xb2, - 0xb4, 0xb6, 0xb8, 0xba, 0xbc, 0xbe, 0xc0, 0xc2, 0xc4, 0xc6, 0xc8, 0xca, - 0xcc, 0xce, 0xd0, 0xd2, 0xd4, 0xd6, 0xd8, 0xda, 0x2, 0x1d, 0x8, 0x2, - 0x5, 0x5, 0x19, 0x19, 0x1c, 0x1c, 0x26, 0x26, 0x65, 0x65, 0xa7, 0xa7, - 0x4, 0x2, 0x10, 0x10, 0x1e, 0x1e, 0x5, 0x2, 0x5, 0x5, 0x26, 0x26, 0x65, - 0x65, 0x4, 0x2, 0x29, 0x29, 0x2b, 0x2b, 0x4, 0x2, 0x2c, 0x2c, 0x32, - 0x32, 0x5, 0x2, 0xf, 0xf, 0x96, 0x96, 0x9c, 0x9c, 0x4, 0x2, 0x20, 0x20, - 0x89, 0x89, 0x4, 0x2, 0x52, 0x52, 0x5e, 0x5e, 0x4, 0x2, 0x45, 0x45, - 0x63, 0x63, 0x5, 0x2, 0x6, 0x6, 0xa, 0xa, 0xe, 0xe, 0x6, 0x2, 0x6, 0x6, - 0x9, 0xa, 0xe, 0xe, 0x8d, 0x8d, 0x4, 0x2, 0x5e, 0x5e, 0x88, 0x88, 0x4, - 0x2, 0x6, 0x6, 0xa, 0xa, 0x4, 0x2, 0x74, 0x74, 0xc4, 0xc4, 0x4, 0x2, - 0xd, 0xd, 0x29, 0x2a, 0x4, 0x2, 0x3d, 0x3d, 0x5b, 0x5b, 0x4, 0x2, 0x42, - 0x42, 0x4e, 0x4e, 0x3, 0x2, 0x93, 0x94, 0x5, 0x2, 0x12, 0x12, 0x5d, - 0x5d, 0xa4, 0xa4, 0x5, 0x2, 0xc0, 0xc0, 0xd2, 0xd2, 0xdb, 0xdb, 0x4, - 0x2, 0xc5, 0xc6, 0xd3, 0xd3, 0x4, 0x2, 0x4d, 0x4d, 0x60, 0x60, 0x3, - 0x2, 0xbb, 0xbc, 0x4, 0x2, 0xc6, 0xc6, 0xd3, 0xd3, 0xa, 0x2, 0x24, 0x24, - 0x4a, 0x4a, 0x6a, 0x6a, 0x6c, 0x6c, 0x80, 0x80, 0x8b, 0x8b, 0xb2, 0xb2, - 0xb6, 0xb6, 0xe, 0x2, 0x4, 0x23, 0x25, 0x49, 0x4b, 0x4f, 0x51, 0x69, - 0x6b, 0x6b, 0x6d, 0x6e, 0x70, 0x71, 0x73, 0x7e, 0x81, 0x8a, 0x8c, 0xb1, - 0xb3, 0xb5, 0xb7, 0xb8, 0x6, 0x2, 0x23, 0x23, 0x3d, 0x3d, 0x4b, 0x4b, - 0x59, 0x59, 0x2, 0x88a, 0x2, 0xea, 0x3, 0x2, 0x2, 0x2, 0x4, 0xfe, 0x3, - 0x2, 0x2, 0x2, 0x6, 0x100, 0x3, 0x2, 0x2, 0x2, 0x8, 0x1e4, 0x3, 0x2, - 0x2, 0x2, 0xa, 0x1e6, 0x3, 0x2, 0x2, 0x2, 0xc, 0x1ee, 0x3, 0x2, 0x2, - 0x2, 0xe, 0x1f2, 0x3, 0x2, 0x2, 0x2, 0x10, 0x1f9, 0x3, 0x2, 0x2, 0x2, - 0x12, 0x1fb, 0x3, 0x2, 0x2, 0x2, 0x14, 0x201, 0x3, 0x2, 0x2, 0x2, 0x16, - 0x290, 0x3, 0x2, 0x2, 0x2, 0x18, 0x292, 0x3, 0x2, 0x2, 0x2, 0x1a, 0x29d, - 0x3, 0x2, 0x2, 0x2, 0x1c, 0x2b8, 0x3, 0x2, 0x2, 0x2, 0x1e, 0x2d3, 0x3, - 0x2, 0x2, 0x2, 0x20, 0x2d7, 0x3, 0x2, 0x2, 0x2, 0x22, 0x2e0, 0x3, 0x2, - 0x2, 0x2, 0x24, 0x2ed, 0x3, 0x2, 0x2, 0x2, 0x26, 0x2fc, 0x3, 0x2, 0x2, - 0x2, 0x28, 0x309, 0x3, 0x2, 0x2, 0x2, 0x2a, 0x319, 0x3, 0x2, 0x2, 0x2, - 0x2c, 0x31e, 0x3, 0x2, 0x2, 0x2, 0x2e, 0x324, 0x3, 0x2, 0x2, 0x2, 0x30, - 0x327, 0x3, 0x2, 0x2, 0x2, 0x32, 0x32a, 0x3, 0x2, 0x2, 0x2, 0x34, 0x33c, - 0x3, 0x2, 0x2, 0x2, 0x36, 0x33e, 0x3, 0x2, 0x2, 0x2, 0x38, 0x35c, 0x3, - 0x2, 0x2, 0x2, 0x3a, 0x360, 0x3, 0x2, 0x2, 0x2, 0x3c, 0x364, 0x3, 0x2, - 0x2, 0x2, 0x3e, 0x368, 0x3, 0x2, 0x2, 0x2, 0x40, 0x371, 0x3, 0x2, 0x2, - 0x2, 0x42, 0x387, 0x3, 0x2, 0x2, 0x2, 0x44, 0x3a9, 0x3, 0x2, 0x2, 0x2, - 0x46, 0x3ab, 0x3, 0x2, 0x2, 0x2, 0x48, 0x3ae, 0x3, 0x2, 0x2, 0x2, 0x4a, - 0x3b5, 0x3, 0x2, 0x2, 0x2, 0x4c, 0x3b8, 0x3, 0x2, 0x2, 0x2, 0x4e, 0x3c4, - 0x3, 0x2, 0x2, 0x2, 0x50, 0x3cc, 0x3, 0x2, 0x2, 0x2, 0x52, 0x3d6, 0x3, - 0x2, 0x2, 0x2, 0x54, 0x3fb, 0x3, 0x2, 0x2, 0x2, 0x56, 0x40a, 0x3, 0x2, - 0x2, 0x2, 0x58, 0x40c, 0x3, 0x2, 0x2, 0x2, 0x5a, 0x410, 0x3, 0x2, 0x2, - 0x2, 0x5c, 0x41f, 0x3, 0x2, 0x2, 0x2, 0x5e, 0x433, 0x3, 0x2, 0x2, 0x2, - 0x60, 0x435, 0x3, 0x2, 0x2, 0x2, 0x62, 0x43e, 0x3, 0x2, 0x2, 0x2, 0x64, - 0x44d, 0x3, 0x2, 0x2, 0x2, 0x66, 0x45f, 0x3, 0x2, 0x2, 0x2, 0x68, 0x46d, - 0x3, 0x2, 0x2, 0x2, 0x6a, 0x47b, 0x3, 0x2, 0x2, 0x2, 0x6c, 0x47e, 0x3, - 0x2, 0x2, 0x2, 0x6e, 0x4ae, 0x3, 0x2, 0x2, 0x2, 0x70, 0x4b1, 0x3, 0x2, - 0x2, 0x2, 0x72, 0x4b7, 0x3, 0x2, 0x2, 0x2, 0x74, 0x4bb, 0x3, 0x2, 0x2, - 0x2, 0x76, 0x4c1, 0x3, 0x2, 0x2, 0x2, 0x78, 0x4c4, 0x3, 0x2, 0x2, 0x2, - 0x7a, 0x4c7, 0x3, 0x2, 0x2, 0x2, 0x7c, 0x4d1, 0x3, 0x2, 0x2, 0x2, 0x7e, - 0x4d4, 0x3, 0x2, 0x2, 0x2, 0x80, 0x4d8, 0x3, 0x2, 0x2, 0x2, 0x82, 0x4dc, - 0x3, 0x2, 0x2, 0x2, 0x84, 0x4e1, 0x3, 0x2, 0x2, 0x2, 0x86, 0x4e7, 0x3, - 0x2, 0x2, 0x2, 0x88, 0x4f6, 0x3, 0x2, 0x2, 0x2, 0x8a, 0x537, 0x3, 0x2, - 0x2, 0x2, 0x8c, 0x53f, 0x3, 0x2, 0x2, 0x2, 0x8e, 0x54a, 0x3, 0x2, 0x2, - 0x2, 0x90, 0x54c, 0x3, 0x2, 0x2, 0x2, 0x92, 0x552, 0x3, 0x2, 0x2, 0x2, - 0x94, 0x557, 0x3, 0x2, 0x2, 0x2, 0x96, 0x55f, 0x3, 0x2, 0x2, 0x2, 0x98, - 0x56b, 0x3, 0x2, 0x2, 0x2, 0x9a, 0x570, 0x3, 0x2, 0x2, 0x2, 0x9c, 0x578, - 0x3, 0x2, 0x2, 0x2, 0x9e, 0x57c, 0x3, 0x2, 0x2, 0x2, 0xa0, 0x5a9, 0x3, - 0x2, 0x2, 0x2, 0xa2, 0x5cd, 0x3, 0x2, 0x2, 0x2, 0xa4, 0x5cf, 0x3, 0x2, - 0x2, 0x2, 0xa6, 0x5de, 0x3, 0x2, 0x2, 0x2, 0xa8, 0x5e1, 0x3, 0x2, 0x2, - 0x2, 0xaa, 0x619, 0x3, 0x2, 0x2, 0x2, 0xac, 0x61b, 0x3, 0x2, 0x2, 0x2, - 0xae, 0x62e, 0x3, 0x2, 0x2, 0x2, 0xb0, 0x69b, 0x3, 0x2, 0x2, 0x2, 0xb2, - 0x6e7, 0x3, 0x2, 0x2, 0x2, 0xb4, 0x6f1, 0x3, 0x2, 0x2, 0x2, 0xb6, 0x706, - 0x3, 0x2, 0x2, 0x2, 0xb8, 0x70e, 0x3, 0x2, 0x2, 0x2, 0xba, 0x712, 0x3, - 0x2, 0x2, 0x2, 0xbc, 0x71e, 0x3, 0x2, 0x2, 0x2, 0xbe, 0x72b, 0x3, 0x2, - 0x2, 0x2, 0xc0, 0x735, 0x3, 0x2, 0x2, 0x2, 0xc2, 0x739, 0x3, 0x2, 0x2, - 0x2, 0xc4, 0x744, 0x3, 0x2, 0x2, 0x2, 0xc6, 0x746, 0x3, 0x2, 0x2, 0x2, - 0xc8, 0x750, 0x3, 0x2, 0x2, 0x2, 0xca, 0x753, 0x3, 0x2, 0x2, 0x2, 0xcc, - 0x760, 0x3, 0x2, 0x2, 0x2, 0xce, 0x762, 0x3, 0x2, 0x2, 0x2, 0xd0, 0x764, - 0x3, 0x2, 0x2, 0x2, 0xd2, 0x766, 0x3, 0x2, 0x2, 0x2, 0xd4, 0x76a, 0x3, - 0x2, 0x2, 0x2, 0xd6, 0x76f, 0x3, 0x2, 0x2, 0x2, 0xd8, 0x773, 0x3, 0x2, - 0x2, 0x2, 0xda, 0x775, 0x3, 0x2, 0x2, 0x2, 0xdc, 0xe0, 0x5, 0x4, 0x3, - 0x2, 0xdd, 0xde, 0x7, 0x55, 0x2, 0x2, 0xde, 0xdf, 0x7, 0x7a, 0x2, 0x2, - 0xdf, 0xe1, 0x7, 0xbe, 0x2, 0x2, 0xe0, 0xdd, 0x3, 0x2, 0x2, 0x2, 0xe0, - 0xe1, 0x3, 0x2, 0x2, 0x2, 0xe1, 0xe4, 0x3, 0x2, 0x2, 0x2, 0xe2, 0xe3, - 0x7, 0x40, 0x2, 0x2, 0xe3, 0xe5, 0x5, 0xd8, 0x6d, 0x2, 0xe4, 0xe2, 0x3, - 0x2, 0x2, 0x2, 0xe4, 0xe5, 0x3, 0x2, 0x2, 0x2, 0xe5, 0xe7, 0x3, 0x2, - 0x2, 0x2, 0xe6, 0xe8, 0x7, 0xda, 0x2, 0x2, 0xe7, 0xe6, 0x3, 0x2, 0x2, - 0x2, 0xe7, 0xe8, 0x3, 0x2, 0x2, 0x2, 0xe8, 0xeb, 0x3, 0x2, 0x2, 0x2, - 0xe9, 0xeb, 0x5, 0x5a, 0x2e, 0x2, 0xea, 0xdc, 0x3, 0x2, 0x2, 0x2, 0xea, - 0xe9, 0x3, 0x2, 0x2, 0x2, 0xeb, 0x3, 0x3, 0x2, 0x2, 0x2, 0xec, 0xff, - 0x5, 0x6, 0x4, 0x2, 0xed, 0xff, 0x5, 0x12, 0xa, 0x2, 0xee, 0xff, 0x5, - 0x14, 0xb, 0x2, 0xef, 0xff, 0x5, 0x16, 0xc, 0x2, 0xf0, 0xff, 0x5, 0x52, - 0x2a, 0x2, 0xf1, 0xff, 0x5, 0x54, 0x2b, 0x2, 0xf2, 0xff, 0x5, 0x56, - 0x2c, 0x2, 0xf3, 0xff, 0x5, 0x58, 0x2d, 0x2, 0xf4, 0xff, 0x5, 0x60, - 0x31, 0x2, 0xf5, 0xff, 0x5, 0x62, 0x32, 0x2, 0xf6, 0xff, 0x5, 0x64, - 0x33, 0x2, 0xf7, 0xff, 0x5, 0x68, 0x35, 0x2, 0xf8, 0xff, 0x5, 0x9e, - 0x50, 0x2, 0xf9, 0xff, 0x5, 0xa0, 0x51, 0x2, 0xfa, 0xff, 0x5, 0xa2, - 0x52, 0x2, 0xfb, 0xff, 0x5, 0xa4, 0x53, 0x2, 0xfc, 0xff, 0x5, 0xa6, - 0x54, 0x2, 0xfd, 0xff, 0x5, 0xa8, 0x55, 0x2, 0xfe, 0xec, 0x3, 0x2, 0x2, - 0x2, 0xfe, 0xed, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xee, 0x3, 0x2, 0x2, 0x2, - 0xfe, 0xef, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf0, 0x3, 0x2, 0x2, 0x2, 0xfe, - 0xf1, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf2, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf3, - 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf4, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf5, 0x3, - 0x2, 0x2, 0x2, 0xfe, 0xf6, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf7, 0x3, 0x2, - 0x2, 0x2, 0xfe, 0xf8, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf9, 0x3, 0x2, 0x2, - 0x2, 0xfe, 0xfa, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xfb, 0x3, 0x2, 0x2, 0x2, - 0xfe, 0xfc, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xfd, 0x3, 0x2, 0x2, 0x2, 0xff, - 0x5, 0x3, 0x2, 0x2, 0x2, 0x100, 0x101, 0x7, 0x7, 0x2, 0x2, 0x101, 0x102, - 0x7, 0x99, 0x2, 0x2, 0x102, 0x104, 0x5, 0xc0, 0x61, 0x2, 0x103, 0x105, + 0x59, 0x5, 0x59, 0x6c6, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, + 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x5, + 0x59, 0x6de, 0xa, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, 0x3, 0x59, + 0x3, 0x59, 0x5, 0x59, 0x6e5, 0xa, 0x59, 0x7, 0x59, 0x6e7, 0xa, 0x59, + 0xc, 0x59, 0xe, 0x59, 0x6ea, 0xb, 0x59, 0x3, 0x5a, 0x3, 0x5a, 0x3, 0x5a, + 0x7, 0x5a, 0x6ef, 0xa, 0x5a, 0xc, 0x5a, 0xe, 0x5a, 0x6f2, 0xb, 0x5a, + 0x3, 0x5b, 0x3, 0x5b, 0x5, 0x5b, 0x6f6, 0xa, 0x5b, 0x3, 0x5c, 0x3, 0x5c, + 0x3, 0x5c, 0x3, 0x5c, 0x7, 0x5c, 0x6fc, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, + 0x6ff, 0xb, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, + 0x7, 0x5c, 0x706, 0xa, 0x5c, 0xc, 0x5c, 0xe, 0x5c, 0x709, 0xb, 0x5c, + 0x5, 0x5c, 0x70b, 0xa, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5c, 0x3, 0x5d, + 0x3, 0x5d, 0x3, 0x5d, 0x5, 0x5d, 0x713, 0xa, 0x5d, 0x3, 0x5d, 0x3, 0x5d, + 0x3, 0x5e, 0x3, 0x5e, 0x3, 0x5e, 0x5, 0x5e, 0x71a, 0xa, 0x5e, 0x3, 0x5f, + 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x5, + 0x5f, 0x723, 0xa, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, 0x3, 0x5f, + 0x5, 0x5f, 0x729, 0xa, 0x5f, 0x7, 0x5f, 0x72b, 0xa, 0x5f, 0xc, 0x5f, + 0xe, 0x5f, 0x72e, 0xb, 0x5f, 0x3, 0x60, 0x3, 0x60, 0x3, 0x60, 0x5, 0x60, + 0x733, 0xa, 0x60, 0x3, 0x60, 0x3, 0x60, 0x3, 0x61, 0x3, 0x61, 0x3, 0x61, + 0x5, 0x61, 0x73a, 0xa, 0x61, 0x3, 0x61, 0x3, 0x61, 0x3, 0x62, 0x3, 0x62, + 0x3, 0x62, 0x7, 0x62, 0x741, 0xa, 0x62, 0xc, 0x62, 0xe, 0x62, 0x744, + 0xb, 0x62, 0x3, 0x63, 0x3, 0x63, 0x3, 0x63, 0x5, 0x63, 0x749, 0xa, 0x63, + 0x3, 0x64, 0x3, 0x64, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, 0x65, 0x3, + 0x65, 0x3, 0x65, 0x5, 0x65, 0x753, 0xa, 0x65, 0x5, 0x65, 0x755, 0xa, + 0x65, 0x3, 0x66, 0x5, 0x66, 0x758, 0xa, 0x66, 0x3, 0x66, 0x3, 0x66, + 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x3, 0x66, 0x5, 0x66, 0x760, 0xa, 0x66, + 0x3, 0x67, 0x3, 0x67, 0x3, 0x67, 0x5, 0x67, 0x765, 0xa, 0x67, 0x3, 0x68, + 0x3, 0x68, 0x3, 0x69, 0x3, 0x69, 0x3, 0x6a, 0x3, 0x6a, 0x3, 0x6b, 0x3, + 0x6b, 0x5, 0x6b, 0x76f, 0xa, 0x6b, 0x3, 0x6c, 0x3, 0x6c, 0x3, 0x6c, + 0x5, 0x6c, 0x774, 0xa, 0x6c, 0x3, 0x6d, 0x3, 0x6d, 0x5, 0x6d, 0x778, + 0xa, 0x6d, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x3, 0x6e, 0x2, + 0x5, 0x88, 0xb0, 0xbc, 0x6f, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10, + 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22, 0x24, 0x26, 0x28, + 0x2a, 0x2c, 0x2e, 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e, 0x40, + 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e, 0x50, 0x52, 0x54, 0x56, 0x58, + 0x5a, 0x5c, 0x5e, 0x60, 0x62, 0x64, 0x66, 0x68, 0x6a, 0x6c, 0x6e, 0x70, + 0x72, 0x74, 0x76, 0x78, 0x7a, 0x7c, 0x7e, 0x80, 0x82, 0x84, 0x86, 0x88, + 0x8a, 0x8c, 0x8e, 0x90, 0x92, 0x94, 0x96, 0x98, 0x9a, 0x9c, 0x9e, 0xa0, + 0xa2, 0xa4, 0xa6, 0xa8, 0xaa, 0xac, 0xae, 0xb0, 0xb2, 0xb4, 0xb6, 0xb8, + 0xba, 0xbc, 0xbe, 0xc0, 0xc2, 0xc4, 0xc6, 0xc8, 0xca, 0xcc, 0xce, 0xd0, + 0xd2, 0xd4, 0xd6, 0xd8, 0xda, 0x2, 0x1d, 0x8, 0x2, 0x5, 0x5, 0x1a, 0x1a, + 0x1d, 0x1d, 0x27, 0x27, 0x67, 0x67, 0xa8, 0xa8, 0x4, 0x2, 0x11, 0x11, + 0x1f, 0x1f, 0x5, 0x2, 0x5, 0x5, 0x27, 0x27, 0x67, 0x67, 0x4, 0x2, 0x2a, + 0x2a, 0x2c, 0x2c, 0x4, 0x2, 0x2d, 0x2d, 0x33, 0x33, 0x5, 0x2, 0x10, + 0x10, 0x97, 0x97, 0x9d, 0x9d, 0x4, 0x2, 0x21, 0x21, 0x8a, 0x8a, 0x4, + 0x2, 0x53, 0x53, 0x5f, 0x5f, 0x4, 0x2, 0x46, 0x46, 0x64, 0x64, 0x5, + 0x2, 0x6, 0x6, 0xa, 0xa, 0xe, 0xe, 0x6, 0x2, 0x6, 0x6, 0x9, 0xa, 0xe, + 0xe, 0x8e, 0x8e, 0x4, 0x2, 0x5f, 0x5f, 0x89, 0x89, 0x4, 0x2, 0x6, 0x6, + 0xa, 0xa, 0x4, 0x2, 0x75, 0x75, 0xc5, 0xc5, 0x4, 0x2, 0xd, 0xd, 0x2a, + 0x2b, 0x4, 0x2, 0x3e, 0x3e, 0x5c, 0x5c, 0x4, 0x2, 0x43, 0x43, 0x4f, + 0x4f, 0x3, 0x2, 0x94, 0x95, 0x5, 0x2, 0x13, 0x13, 0x5e, 0x5e, 0xa5, + 0xa5, 0x5, 0x2, 0xc1, 0xc1, 0xd3, 0xd3, 0xdc, 0xdc, 0x4, 0x2, 0xc6, + 0xc7, 0xd4, 0xd4, 0x4, 0x2, 0x4e, 0x4e, 0x61, 0x61, 0x3, 0x2, 0xbc, + 0xbd, 0x4, 0x2, 0xc7, 0xc7, 0xd4, 0xd4, 0xa, 0x2, 0x25, 0x25, 0x4b, + 0x4b, 0x6b, 0x6b, 0x6d, 0x6d, 0x81, 0x81, 0x8c, 0x8c, 0xb3, 0xb3, 0xb7, + 0xb7, 0xe, 0x2, 0x4, 0x24, 0x26, 0x4a, 0x4c, 0x50, 0x52, 0x6a, 0x6c, + 0x6c, 0x6e, 0x6f, 0x71, 0x72, 0x74, 0x7f, 0x82, 0x8b, 0x8d, 0xb2, 0xb4, + 0xb6, 0xb8, 0xb9, 0x6, 0x2, 0x24, 0x24, 0x3e, 0x3e, 0x4c, 0x4c, 0x5a, + 0x5a, 0x2, 0x88f, 0x2, 0xea, 0x3, 0x2, 0x2, 0x2, 0x4, 0xfe, 0x3, 0x2, + 0x2, 0x2, 0x6, 0x100, 0x3, 0x2, 0x2, 0x2, 0x8, 0x1e4, 0x3, 0x2, 0x2, + 0x2, 0xa, 0x1e6, 0x3, 0x2, 0x2, 0x2, 0xc, 0x1ee, 0x3, 0x2, 0x2, 0x2, + 0xe, 0x1f2, 0x3, 0x2, 0x2, 0x2, 0x10, 0x1f9, 0x3, 0x2, 0x2, 0x2, 0x12, + 0x1fb, 0x3, 0x2, 0x2, 0x2, 0x14, 0x201, 0x3, 0x2, 0x2, 0x2, 0x16, 0x290, + 0x3, 0x2, 0x2, 0x2, 0x18, 0x292, 0x3, 0x2, 0x2, 0x2, 0x1a, 0x29d, 0x3, + 0x2, 0x2, 0x2, 0x1c, 0x2b8, 0x3, 0x2, 0x2, 0x2, 0x1e, 0x2d3, 0x3, 0x2, + 0x2, 0x2, 0x20, 0x2d7, 0x3, 0x2, 0x2, 0x2, 0x22, 0x2e0, 0x3, 0x2, 0x2, + 0x2, 0x24, 0x2ed, 0x3, 0x2, 0x2, 0x2, 0x26, 0x2fc, 0x3, 0x2, 0x2, 0x2, + 0x28, 0x309, 0x3, 0x2, 0x2, 0x2, 0x2a, 0x319, 0x3, 0x2, 0x2, 0x2, 0x2c, + 0x31e, 0x3, 0x2, 0x2, 0x2, 0x2e, 0x324, 0x3, 0x2, 0x2, 0x2, 0x30, 0x327, + 0x3, 0x2, 0x2, 0x2, 0x32, 0x32a, 0x3, 0x2, 0x2, 0x2, 0x34, 0x33c, 0x3, + 0x2, 0x2, 0x2, 0x36, 0x33e, 0x3, 0x2, 0x2, 0x2, 0x38, 0x35c, 0x3, 0x2, + 0x2, 0x2, 0x3a, 0x360, 0x3, 0x2, 0x2, 0x2, 0x3c, 0x364, 0x3, 0x2, 0x2, + 0x2, 0x3e, 0x368, 0x3, 0x2, 0x2, 0x2, 0x40, 0x371, 0x3, 0x2, 0x2, 0x2, + 0x42, 0x387, 0x3, 0x2, 0x2, 0x2, 0x44, 0x3a9, 0x3, 0x2, 0x2, 0x2, 0x46, + 0x3ab, 0x3, 0x2, 0x2, 0x2, 0x48, 0x3ae, 0x3, 0x2, 0x2, 0x2, 0x4a, 0x3b5, + 0x3, 0x2, 0x2, 0x2, 0x4c, 0x3b8, 0x3, 0x2, 0x2, 0x2, 0x4e, 0x3c4, 0x3, + 0x2, 0x2, 0x2, 0x50, 0x3cc, 0x3, 0x2, 0x2, 0x2, 0x52, 0x3d6, 0x3, 0x2, + 0x2, 0x2, 0x54, 0x3fb, 0x3, 0x2, 0x2, 0x2, 0x56, 0x40a, 0x3, 0x2, 0x2, + 0x2, 0x58, 0x412, 0x3, 0x2, 0x2, 0x2, 0x5a, 0x414, 0x3, 0x2, 0x2, 0x2, + 0x5c, 0x423, 0x3, 0x2, 0x2, 0x2, 0x5e, 0x437, 0x3, 0x2, 0x2, 0x2, 0x60, + 0x439, 0x3, 0x2, 0x2, 0x2, 0x62, 0x442, 0x3, 0x2, 0x2, 0x2, 0x64, 0x451, + 0x3, 0x2, 0x2, 0x2, 0x66, 0x463, 0x3, 0x2, 0x2, 0x2, 0x68, 0x471, 0x3, + 0x2, 0x2, 0x2, 0x6a, 0x47f, 0x3, 0x2, 0x2, 0x2, 0x6c, 0x482, 0x3, 0x2, + 0x2, 0x2, 0x6e, 0x4b2, 0x3, 0x2, 0x2, 0x2, 0x70, 0x4b5, 0x3, 0x2, 0x2, + 0x2, 0x72, 0x4bb, 0x3, 0x2, 0x2, 0x2, 0x74, 0x4bf, 0x3, 0x2, 0x2, 0x2, + 0x76, 0x4c5, 0x3, 0x2, 0x2, 0x2, 0x78, 0x4c8, 0x3, 0x2, 0x2, 0x2, 0x7a, + 0x4cb, 0x3, 0x2, 0x2, 0x2, 0x7c, 0x4d5, 0x3, 0x2, 0x2, 0x2, 0x7e, 0x4d8, + 0x3, 0x2, 0x2, 0x2, 0x80, 0x4dc, 0x3, 0x2, 0x2, 0x2, 0x82, 0x4e0, 0x3, + 0x2, 0x2, 0x2, 0x84, 0x4e5, 0x3, 0x2, 0x2, 0x2, 0x86, 0x4eb, 0x3, 0x2, + 0x2, 0x2, 0x88, 0x4fa, 0x3, 0x2, 0x2, 0x2, 0x8a, 0x53b, 0x3, 0x2, 0x2, + 0x2, 0x8c, 0x543, 0x3, 0x2, 0x2, 0x2, 0x8e, 0x54e, 0x3, 0x2, 0x2, 0x2, + 0x90, 0x550, 0x3, 0x2, 0x2, 0x2, 0x92, 0x556, 0x3, 0x2, 0x2, 0x2, 0x94, + 0x55b, 0x3, 0x2, 0x2, 0x2, 0x96, 0x563, 0x3, 0x2, 0x2, 0x2, 0x98, 0x56f, + 0x3, 0x2, 0x2, 0x2, 0x9a, 0x574, 0x3, 0x2, 0x2, 0x2, 0x9c, 0x57c, 0x3, + 0x2, 0x2, 0x2, 0x9e, 0x580, 0x3, 0x2, 0x2, 0x2, 0xa0, 0x5ad, 0x3, 0x2, + 0x2, 0x2, 0xa2, 0x5d1, 0x3, 0x2, 0x2, 0x2, 0xa4, 0x5d3, 0x3, 0x2, 0x2, + 0x2, 0xa6, 0x5e2, 0x3, 0x2, 0x2, 0x2, 0xa8, 0x5e5, 0x3, 0x2, 0x2, 0x2, + 0xaa, 0x61d, 0x3, 0x2, 0x2, 0x2, 0xac, 0x61f, 0x3, 0x2, 0x2, 0x2, 0xae, + 0x632, 0x3, 0x2, 0x2, 0x2, 0xb0, 0x69f, 0x3, 0x2, 0x2, 0x2, 0xb2, 0x6eb, + 0x3, 0x2, 0x2, 0x2, 0xb4, 0x6f5, 0x3, 0x2, 0x2, 0x2, 0xb6, 0x70a, 0x3, + 0x2, 0x2, 0x2, 0xb8, 0x712, 0x3, 0x2, 0x2, 0x2, 0xba, 0x716, 0x3, 0x2, + 0x2, 0x2, 0xbc, 0x722, 0x3, 0x2, 0x2, 0x2, 0xbe, 0x72f, 0x3, 0x2, 0x2, + 0x2, 0xc0, 0x739, 0x3, 0x2, 0x2, 0x2, 0xc2, 0x73d, 0x3, 0x2, 0x2, 0x2, + 0xc4, 0x748, 0x3, 0x2, 0x2, 0x2, 0xc6, 0x74a, 0x3, 0x2, 0x2, 0x2, 0xc8, + 0x754, 0x3, 0x2, 0x2, 0x2, 0xca, 0x757, 0x3, 0x2, 0x2, 0x2, 0xcc, 0x764, + 0x3, 0x2, 0x2, 0x2, 0xce, 0x766, 0x3, 0x2, 0x2, 0x2, 0xd0, 0x768, 0x3, + 0x2, 0x2, 0x2, 0xd2, 0x76a, 0x3, 0x2, 0x2, 0x2, 0xd4, 0x76e, 0x3, 0x2, + 0x2, 0x2, 0xd6, 0x773, 0x3, 0x2, 0x2, 0x2, 0xd8, 0x777, 0x3, 0x2, 0x2, + 0x2, 0xda, 0x779, 0x3, 0x2, 0x2, 0x2, 0xdc, 0xe0, 0x5, 0x4, 0x3, 0x2, + 0xdd, 0xde, 0x7, 0x56, 0x2, 0x2, 0xde, 0xdf, 0x7, 0x7b, 0x2, 0x2, 0xdf, + 0xe1, 0x7, 0xbf, 0x2, 0x2, 0xe0, 0xdd, 0x3, 0x2, 0x2, 0x2, 0xe0, 0xe1, + 0x3, 0x2, 0x2, 0x2, 0xe1, 0xe4, 0x3, 0x2, 0x2, 0x2, 0xe2, 0xe3, 0x7, + 0x41, 0x2, 0x2, 0xe3, 0xe5, 0x5, 0xd8, 0x6d, 0x2, 0xe4, 0xe2, 0x3, 0x2, + 0x2, 0x2, 0xe4, 0xe5, 0x3, 0x2, 0x2, 0x2, 0xe5, 0xe7, 0x3, 0x2, 0x2, + 0x2, 0xe6, 0xe8, 0x7, 0xdb, 0x2, 0x2, 0xe7, 0xe6, 0x3, 0x2, 0x2, 0x2, + 0xe7, 0xe8, 0x3, 0x2, 0x2, 0x2, 0xe8, 0xeb, 0x3, 0x2, 0x2, 0x2, 0xe9, + 0xeb, 0x5, 0x5a, 0x2e, 0x2, 0xea, 0xdc, 0x3, 0x2, 0x2, 0x2, 0xea, 0xe9, + 0x3, 0x2, 0x2, 0x2, 0xeb, 0x3, 0x3, 0x2, 0x2, 0x2, 0xec, 0xff, 0x5, + 0x6, 0x4, 0x2, 0xed, 0xff, 0x5, 0x12, 0xa, 0x2, 0xee, 0xff, 0x5, 0x14, + 0xb, 0x2, 0xef, 0xff, 0x5, 0x16, 0xc, 0x2, 0xf0, 0xff, 0x5, 0x52, 0x2a, + 0x2, 0xf1, 0xff, 0x5, 0x54, 0x2b, 0x2, 0xf2, 0xff, 0x5, 0x56, 0x2c, + 0x2, 0xf3, 0xff, 0x5, 0x58, 0x2d, 0x2, 0xf4, 0xff, 0x5, 0x60, 0x31, + 0x2, 0xf5, 0xff, 0x5, 0x62, 0x32, 0x2, 0xf6, 0xff, 0x5, 0x64, 0x33, + 0x2, 0xf7, 0xff, 0x5, 0x68, 0x35, 0x2, 0xf8, 0xff, 0x5, 0x9e, 0x50, + 0x2, 0xf9, 0xff, 0x5, 0xa0, 0x51, 0x2, 0xfa, 0xff, 0x5, 0xa2, 0x52, + 0x2, 0xfb, 0xff, 0x5, 0xa4, 0x53, 0x2, 0xfc, 0xff, 0x5, 0xa6, 0x54, + 0x2, 0xfd, 0xff, 0x5, 0xa8, 0x55, 0x2, 0xfe, 0xec, 0x3, 0x2, 0x2, 0x2, + 0xfe, 0xed, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xee, 0x3, 0x2, 0x2, 0x2, 0xfe, + 0xef, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf0, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf1, + 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf2, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf3, 0x3, + 0x2, 0x2, 0x2, 0xfe, 0xf4, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf5, 0x3, 0x2, + 0x2, 0x2, 0xfe, 0xf6, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf7, 0x3, 0x2, 0x2, + 0x2, 0xfe, 0xf8, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xf9, 0x3, 0x2, 0x2, 0x2, + 0xfe, 0xfa, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xfb, 0x3, 0x2, 0x2, 0x2, 0xfe, + 0xfc, 0x3, 0x2, 0x2, 0x2, 0xfe, 0xfd, 0x3, 0x2, 0x2, 0x2, 0xff, 0x5, + 0x3, 0x2, 0x2, 0x2, 0x100, 0x101, 0x7, 0x7, 0x2, 0x2, 0x101, 0x102, + 0x7, 0x9a, 0x2, 0x2, 0x102, 0x104, 0x5, 0xc0, 0x61, 0x2, 0x103, 0x105, 0x5, 0x2c, 0x17, 0x2, 0x104, 0x103, 0x3, 0x2, 0x2, 0x2, 0x104, 0x105, 0x3, 0x2, 0x2, 0x2, 0x105, 0x106, 0x3, 0x2, 0x2, 0x2, 0x106, 0x10b, - 0x5, 0x8, 0x5, 0x2, 0x107, 0x108, 0x7, 0xc4, 0x2, 0x2, 0x108, 0x10a, + 0x5, 0x8, 0x5, 0x2, 0x107, 0x108, 0x7, 0xc5, 0x2, 0x2, 0x108, 0x10a, 0x5, 0x8, 0x5, 0x2, 0x109, 0x107, 0x3, 0x2, 0x2, 0x2, 0x10a, 0x10d, 0x3, 0x2, 0x2, 0x2, 0x10b, 0x109, 0x3, 0x2, 0x2, 0x2, 0x10b, 0x10c, 0x3, 0x2, 0x2, 0x2, 0x10c, 0x7, 0x3, 0x2, 0x2, 0x2, 0x10d, 0x10b, 0x3, 0x2, 0x2, 0x2, 0x10e, 0x10f, 0x7, 0x3, 0x2, 0x2, 0x10f, 0x113, 0x7, - 0x1b, 0x2, 0x2, 0x110, 0x111, 0x7, 0x4c, 0x2, 0x2, 0x111, 0x112, 0x7, - 0x71, 0x2, 0x2, 0x112, 0x114, 0x7, 0x37, 0x2, 0x2, 0x113, 0x110, 0x3, + 0x1c, 0x2, 0x2, 0x110, 0x111, 0x7, 0x4d, 0x2, 0x2, 0x111, 0x112, 0x7, + 0x72, 0x2, 0x2, 0x112, 0x114, 0x7, 0x38, 0x2, 0x2, 0x113, 0x110, 0x3, 0x2, 0x2, 0x2, 0x113, 0x114, 0x3, 0x2, 0x2, 0x2, 0x114, 0x115, 0x3, 0x2, 0x2, 0x2, 0x115, 0x118, 0x5, 0x44, 0x23, 0x2, 0x116, 0x117, 0x7, 0x4, 0x2, 0x2, 0x117, 0x119, 0x5, 0xba, 0x5e, 0x2, 0x118, 0x116, 0x3, 0x2, 0x2, 0x2, 0x118, 0x119, 0x3, 0x2, 0x2, 0x2, 0x119, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x11a, 0x11b, 0x7, 0x3, 0x2, 0x2, 0x11b, 0x11f, 0x7, - 0x4f, 0x2, 0x2, 0x11c, 0x11d, 0x7, 0x4c, 0x2, 0x2, 0x11d, 0x11e, 0x7, - 0x71, 0x2, 0x2, 0x11e, 0x120, 0x7, 0x37, 0x2, 0x2, 0x11f, 0x11c, 0x3, + 0x50, 0x2, 0x2, 0x11c, 0x11d, 0x7, 0x4d, 0x2, 0x2, 0x11d, 0x11e, 0x7, + 0x72, 0x2, 0x2, 0x11e, 0x120, 0x7, 0x38, 0x2, 0x2, 0x11f, 0x11c, 0x3, 0x2, 0x2, 0x2, 0x11f, 0x120, 0x3, 0x2, 0x2, 0x2, 0x120, 0x121, 0x3, 0x2, 0x2, 0x2, 0x121, 0x124, 0x5, 0x48, 0x25, 0x2, 0x122, 0x123, 0x7, 0x4, 0x2, 0x2, 0x123, 0x125, 0x5, 0xba, 0x5e, 0x2, 0x124, 0x122, 0x3, 0x2, 0x2, 0x2, 0x124, 0x125, 0x3, 0x2, 0x2, 0x2, 0x125, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x126, 0x127, 0x7, 0x3, 0x2, 0x2, 0x127, 0x12b, 0x7, - 0x7f, 0x2, 0x2, 0x128, 0x129, 0x7, 0x4c, 0x2, 0x2, 0x129, 0x12a, 0x7, - 0x71, 0x2, 0x2, 0x12a, 0x12c, 0x7, 0x37, 0x2, 0x2, 0x12b, 0x128, 0x3, + 0x80, 0x2, 0x2, 0x128, 0x129, 0x7, 0x4d, 0x2, 0x2, 0x129, 0x12a, 0x7, + 0x72, 0x2, 0x2, 0x12a, 0x12c, 0x7, 0x38, 0x2, 0x2, 0x12b, 0x128, 0x3, 0x2, 0x2, 0x2, 0x12b, 0x12c, 0x3, 0x2, 0x2, 0x2, 0x12c, 0x12d, 0x3, 0x2, 0x2, 0x2, 0x12d, 0x130, 0x5, 0x4a, 0x26, 0x2, 0x12e, 0x12f, 0x7, 0x4, 0x2, 0x2, 0x12f, 0x131, 0x5, 0xba, 0x5e, 0x2, 0x130, 0x12e, 0x3, 0x2, 0x2, 0x2, 0x130, 0x131, 0x3, 0x2, 0x2, 0x2, 0x131, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x132, 0x133, 0x7, 0x10, 0x2, 0x2, 0x133, 0x136, 0x5, - 0x10, 0x9, 0x2, 0x134, 0x135, 0x7, 0x42, 0x2, 0x2, 0x135, 0x137, 0x5, + 0x2, 0x2, 0x2, 0x132, 0x133, 0x7, 0x11, 0x2, 0x2, 0x133, 0x136, 0x5, + 0x10, 0x9, 0x2, 0x134, 0x135, 0x7, 0x43, 0x2, 0x2, 0x135, 0x137, 0x5, 0xc0, 0x61, 0x2, 0x136, 0x134, 0x3, 0x2, 0x2, 0x2, 0x136, 0x137, 0x3, 0x2, 0x2, 0x2, 0x137, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x138, 0x139, 0x7, - 0x17, 0x2, 0x2, 0x139, 0x13c, 0x7, 0x1b, 0x2, 0x2, 0x13a, 0x13b, 0x7, - 0x4c, 0x2, 0x2, 0x13b, 0x13d, 0x7, 0x37, 0x2, 0x2, 0x13c, 0x13a, 0x3, + 0x18, 0x2, 0x2, 0x139, 0x13c, 0x7, 0x1c, 0x2, 0x2, 0x13a, 0x13b, 0x7, + 0x4d, 0x2, 0x2, 0x13b, 0x13d, 0x7, 0x38, 0x2, 0x2, 0x13c, 0x13a, 0x3, 0x2, 0x2, 0x2, 0x13c, 0x13d, 0x3, 0x2, 0x2, 0x2, 0x13d, 0x13e, 0x3, 0x2, 0x2, 0x2, 0x13e, 0x141, 0x5, 0xba, 0x5e, 0x2, 0x13f, 0x140, 0x7, - 0x4e, 0x2, 0x2, 0x140, 0x142, 0x5, 0x10, 0x9, 0x2, 0x141, 0x13f, 0x3, + 0x4f, 0x2, 0x2, 0x140, 0x142, 0x5, 0x10, 0x9, 0x2, 0x141, 0x13f, 0x3, 0x2, 0x2, 0x2, 0x141, 0x142, 0x3, 0x2, 0x2, 0x2, 0x142, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x143, 0x144, 0x7, 0x17, 0x2, 0x2, 0x144, 0x147, 0x7, - 0x4f, 0x2, 0x2, 0x145, 0x146, 0x7, 0x4c, 0x2, 0x2, 0x146, 0x148, 0x7, - 0x37, 0x2, 0x2, 0x147, 0x145, 0x3, 0x2, 0x2, 0x2, 0x147, 0x148, 0x3, + 0x2, 0x2, 0x2, 0x143, 0x144, 0x7, 0x18, 0x2, 0x2, 0x144, 0x147, 0x7, + 0x50, 0x2, 0x2, 0x145, 0x146, 0x7, 0x4d, 0x2, 0x2, 0x146, 0x148, 0x7, + 0x38, 0x2, 0x2, 0x147, 0x145, 0x3, 0x2, 0x2, 0x2, 0x147, 0x148, 0x3, 0x2, 0x2, 0x2, 0x148, 0x149, 0x3, 0x2, 0x2, 0x2, 0x149, 0x14c, 0x5, - 0xba, 0x5e, 0x2, 0x14a, 0x14b, 0x7, 0x4e, 0x2, 0x2, 0x14b, 0x14d, 0x5, + 0xba, 0x5e, 0x2, 0x14a, 0x14b, 0x7, 0x4f, 0x2, 0x2, 0x14b, 0x14d, 0x5, 0x10, 0x9, 0x2, 0x14c, 0x14a, 0x3, 0x2, 0x2, 0x2, 0x14c, 0x14d, 0x3, 0x2, 0x2, 0x2, 0x14d, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x14e, 0x14f, 0x7, - 0x17, 0x2, 0x2, 0x14f, 0x152, 0x7, 0x7f, 0x2, 0x2, 0x150, 0x151, 0x7, - 0x4c, 0x2, 0x2, 0x151, 0x153, 0x7, 0x37, 0x2, 0x2, 0x152, 0x150, 0x3, + 0x18, 0x2, 0x2, 0x14f, 0x152, 0x7, 0x80, 0x2, 0x2, 0x150, 0x151, 0x7, + 0x4d, 0x2, 0x2, 0x151, 0x153, 0x7, 0x38, 0x2, 0x2, 0x152, 0x150, 0x3, 0x2, 0x2, 0x2, 0x152, 0x153, 0x3, 0x2, 0x2, 0x2, 0x153, 0x154, 0x3, 0x2, 0x2, 0x2, 0x154, 0x157, 0x5, 0xba, 0x5e, 0x2, 0x155, 0x156, 0x7, - 0x4e, 0x2, 0x2, 0x156, 0x158, 0x5, 0x10, 0x9, 0x2, 0x157, 0x155, 0x3, + 0x4f, 0x2, 0x2, 0x156, 0x158, 0x5, 0x10, 0x9, 0x2, 0x157, 0x155, 0x3, 0x2, 0x2, 0x2, 0x157, 0x158, 0x3, 0x2, 0x2, 0x2, 0x158, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x159, 0x15a, 0x7, 0x1c, 0x2, 0x2, 0x15a, 0x15d, 0x7, - 0x1b, 0x2, 0x2, 0x15b, 0x15c, 0x7, 0x4c, 0x2, 0x2, 0x15c, 0x15e, 0x7, - 0x37, 0x2, 0x2, 0x15d, 0x15b, 0x3, 0x2, 0x2, 0x2, 0x15d, 0x15e, 0x3, + 0x2, 0x2, 0x2, 0x159, 0x15a, 0x7, 0x1d, 0x2, 0x2, 0x15a, 0x15d, 0x7, + 0x1c, 0x2, 0x2, 0x15b, 0x15c, 0x7, 0x4d, 0x2, 0x2, 0x15c, 0x15e, 0x7, + 0x38, 0x2, 0x2, 0x15d, 0x15b, 0x3, 0x2, 0x2, 0x2, 0x15d, 0x15e, 0x3, 0x2, 0x2, 0x2, 0x15e, 0x15f, 0x3, 0x2, 0x2, 0x2, 0x15f, 0x160, 0x5, - 0xba, 0x5e, 0x2, 0x160, 0x161, 0x7, 0xbe, 0x2, 0x2, 0x161, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x162, 0x163, 0x7, 0x28, 0x2, 0x2, 0x163, 0x164, 0x7, - 0xb4, 0x2, 0x2, 0x164, 0x1e5, 0x5, 0xb0, 0x59, 0x2, 0x165, 0x166, 0x7, - 0x2c, 0x2, 0x2, 0x166, 0x1e5, 0x5, 0x10, 0x9, 0x2, 0x167, 0x168, 0x7, - 0x32, 0x2, 0x2, 0x168, 0x16b, 0x7, 0x1b, 0x2, 0x2, 0x169, 0x16a, 0x7, - 0x4c, 0x2, 0x2, 0x16a, 0x16c, 0x7, 0x37, 0x2, 0x2, 0x16b, 0x169, 0x3, + 0xba, 0x5e, 0x2, 0x160, 0x161, 0x7, 0xbf, 0x2, 0x2, 0x161, 0x1e5, 0x3, + 0x2, 0x2, 0x2, 0x162, 0x163, 0x7, 0x29, 0x2, 0x2, 0x163, 0x164, 0x7, + 0xb5, 0x2, 0x2, 0x164, 0x1e5, 0x5, 0xb0, 0x59, 0x2, 0x165, 0x166, 0x7, + 0x2d, 0x2, 0x2, 0x166, 0x1e5, 0x5, 0x10, 0x9, 0x2, 0x167, 0x168, 0x7, + 0x33, 0x2, 0x2, 0x168, 0x16b, 0x7, 0x1c, 0x2, 0x2, 0x169, 0x16a, 0x7, + 0x4d, 0x2, 0x2, 0x16a, 0x16c, 0x7, 0x38, 0x2, 0x2, 0x16b, 0x169, 0x3, 0x2, 0x2, 0x2, 0x16b, 0x16c, 0x3, 0x2, 0x2, 0x2, 0x16c, 0x16d, 0x3, 0x2, 0x2, 0x2, 0x16d, 0x1e5, 0x5, 0xba, 0x5e, 0x2, 0x16e, 0x16f, 0x7, - 0x32, 0x2, 0x2, 0x16f, 0x172, 0x7, 0x4f, 0x2, 0x2, 0x170, 0x171, 0x7, - 0x4c, 0x2, 0x2, 0x171, 0x173, 0x7, 0x37, 0x2, 0x2, 0x172, 0x170, 0x3, + 0x33, 0x2, 0x2, 0x16f, 0x172, 0x7, 0x50, 0x2, 0x2, 0x170, 0x171, 0x7, + 0x4d, 0x2, 0x2, 0x171, 0x173, 0x7, 0x38, 0x2, 0x2, 0x172, 0x170, 0x3, 0x2, 0x2, 0x2, 0x172, 0x173, 0x3, 0x2, 0x2, 0x2, 0x173, 0x174, 0x3, 0x2, 0x2, 0x2, 0x174, 0x1e5, 0x5, 0xba, 0x5e, 0x2, 0x175, 0x176, 0x7, - 0x32, 0x2, 0x2, 0x176, 0x179, 0x7, 0x7f, 0x2, 0x2, 0x177, 0x178, 0x7, - 0x4c, 0x2, 0x2, 0x178, 0x17a, 0x7, 0x37, 0x2, 0x2, 0x179, 0x177, 0x3, + 0x33, 0x2, 0x2, 0x176, 0x179, 0x7, 0x80, 0x2, 0x2, 0x177, 0x178, 0x7, + 0x4d, 0x2, 0x2, 0x178, 0x17a, 0x7, 0x38, 0x2, 0x2, 0x179, 0x177, 0x3, 0x2, 0x2, 0x2, 0x179, 0x17a, 0x3, 0x2, 0x2, 0x2, 0x17a, 0x17b, 0x3, 0x2, 0x2, 0x2, 0x17b, 0x1e5, 0x5, 0xba, 0x5e, 0x2, 0x17c, 0x17d, 0x7, - 0x32, 0x2, 0x2, 0x17d, 0x1e5, 0x5, 0x10, 0x9, 0x2, 0x17e, 0x180, 0x7, - 0x41, 0x2, 0x2, 0x17f, 0x181, 0x5, 0x10, 0x9, 0x2, 0x180, 0x17f, 0x3, + 0x33, 0x2, 0x2, 0x17d, 0x1e5, 0x5, 0x10, 0x9, 0x2, 0x17e, 0x180, 0x7, + 0x42, 0x2, 0x2, 0x17f, 0x181, 0x5, 0x10, 0x9, 0x2, 0x180, 0x17f, 0x3, 0x2, 0x2, 0x2, 0x180, 0x181, 0x3, 0x2, 0x2, 0x2, 0x181, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x182, 0x183, 0x7, 0x66, 0x2, 0x2, 0x183, 0x186, 0x7, - 0x4f, 0x2, 0x2, 0x184, 0x185, 0x7, 0x4c, 0x2, 0x2, 0x185, 0x187, 0x7, - 0x37, 0x2, 0x2, 0x186, 0x184, 0x3, 0x2, 0x2, 0x2, 0x186, 0x187, 0x3, + 0x50, 0x2, 0x2, 0x184, 0x185, 0x7, 0x4d, 0x2, 0x2, 0x185, 0x187, 0x7, + 0x38, 0x2, 0x2, 0x186, 0x184, 0x3, 0x2, 0x2, 0x2, 0x186, 0x187, 0x3, 0x2, 0x2, 0x2, 0x187, 0x188, 0x3, 0x2, 0x2, 0x2, 0x188, 0x18b, 0x5, - 0xba, 0x5e, 0x2, 0x189, 0x18a, 0x7, 0x4e, 0x2, 0x2, 0x18a, 0x18c, 0x5, + 0xba, 0x5e, 0x2, 0x189, 0x18a, 0x7, 0x4f, 0x2, 0x2, 0x18a, 0x18c, 0x5, 0x10, 0x9, 0x2, 0x18b, 0x189, 0x3, 0x2, 0x2, 0x2, 0x18b, 0x18c, 0x3, 0x2, 0x2, 0x2, 0x18c, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x18d, 0x18e, 0x7, - 0x66, 0x2, 0x2, 0x18e, 0x191, 0x7, 0x7f, 0x2, 0x2, 0x18f, 0x190, 0x7, - 0x4c, 0x2, 0x2, 0x190, 0x192, 0x7, 0x37, 0x2, 0x2, 0x191, 0x18f, 0x3, + 0x66, 0x2, 0x2, 0x18e, 0x191, 0x7, 0x80, 0x2, 0x2, 0x18f, 0x190, 0x7, + 0x4d, 0x2, 0x2, 0x190, 0x192, 0x7, 0x38, 0x2, 0x2, 0x191, 0x18f, 0x3, 0x2, 0x2, 0x2, 0x191, 0x192, 0x3, 0x2, 0x2, 0x2, 0x192, 0x193, 0x3, 0x2, 0x2, 0x2, 0x193, 0x196, 0x5, 0xba, 0x5e, 0x2, 0x194, 0x195, 0x7, - 0x4e, 0x2, 0x2, 0x195, 0x197, 0x5, 0x10, 0x9, 0x2, 0x196, 0x194, 0x3, + 0x4f, 0x2, 0x2, 0x195, 0x197, 0x5, 0x10, 0x9, 0x2, 0x196, 0x194, 0x3, 0x2, 0x2, 0x2, 0x196, 0x197, 0x3, 0x2, 0x2, 0x2, 0x197, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x198, 0x199, 0x7, 0x6b, 0x2, 0x2, 0x199, 0x19c, 0x7, - 0x1b, 0x2, 0x2, 0x19a, 0x19b, 0x7, 0x4c, 0x2, 0x2, 0x19b, 0x19d, 0x7, - 0x37, 0x2, 0x2, 0x19c, 0x19a, 0x3, 0x2, 0x2, 0x2, 0x19c, 0x19d, 0x3, + 0x2, 0x2, 0x2, 0x198, 0x199, 0x7, 0x6c, 0x2, 0x2, 0x199, 0x19c, 0x7, + 0x1c, 0x2, 0x2, 0x19a, 0x19b, 0x7, 0x4d, 0x2, 0x2, 0x19b, 0x19d, 0x7, + 0x38, 0x2, 0x2, 0x19c, 0x19a, 0x3, 0x2, 0x2, 0x2, 0x19c, 0x19d, 0x3, 0x2, 0x2, 0x2, 0x19d, 0x19e, 0x3, 0x2, 0x2, 0x2, 0x19e, 0x19f, 0x5, 0xba, 0x5e, 0x2, 0x19f, 0x1a0, 0x5, 0x4c, 0x27, 0x2, 0x1a0, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x1a1, 0x1a2, 0x7, 0x6b, 0x2, 0x2, 0x1a2, 0x1a5, 0x7, - 0x1b, 0x2, 0x2, 0x1a3, 0x1a4, 0x7, 0x4c, 0x2, 0x2, 0x1a4, 0x1a6, 0x7, - 0x37, 0x2, 0x2, 0x1a5, 0x1a3, 0x3, 0x2, 0x2, 0x2, 0x1a5, 0x1a6, 0x3, + 0x2, 0x2, 0x2, 0x1a1, 0x1a2, 0x7, 0x6c, 0x2, 0x2, 0x1a2, 0x1a5, 0x7, + 0x1c, 0x2, 0x2, 0x1a3, 0x1a4, 0x7, 0x4d, 0x2, 0x2, 0x1a4, 0x1a6, 0x7, + 0x38, 0x2, 0x2, 0x1a5, 0x1a3, 0x3, 0x2, 0x2, 0x2, 0x1a5, 0x1a6, 0x3, 0x2, 0x2, 0x2, 0x1a6, 0x1a7, 0x3, 0x2, 0x2, 0x2, 0x1a7, 0x1a8, 0x5, - 0xba, 0x5e, 0x2, 0x1a8, 0x1a9, 0x7, 0x1c, 0x2, 0x2, 0x1a9, 0x1aa, 0x7, - 0xbe, 0x2, 0x2, 0x1aa, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x1ab, 0x1ac, 0x7, - 0x6b, 0x2, 0x2, 0x1ac, 0x1af, 0x7, 0x1b, 0x2, 0x2, 0x1ad, 0x1ae, 0x7, - 0x4c, 0x2, 0x2, 0x1ae, 0x1b0, 0x7, 0x37, 0x2, 0x2, 0x1af, 0x1ad, 0x3, + 0xba, 0x5e, 0x2, 0x1a8, 0x1a9, 0x7, 0x1d, 0x2, 0x2, 0x1a9, 0x1aa, 0x7, + 0xbf, 0x2, 0x2, 0x1aa, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x1ab, 0x1ac, 0x7, + 0x6c, 0x2, 0x2, 0x1ac, 0x1af, 0x7, 0x1c, 0x2, 0x2, 0x1ad, 0x1ae, 0x7, + 0x4d, 0x2, 0x2, 0x1ae, 0x1b0, 0x7, 0x38, 0x2, 0x2, 0x1af, 0x1ad, 0x3, 0x2, 0x2, 0x2, 0x1af, 0x1b0, 0x3, 0x2, 0x2, 0x2, 0x1b0, 0x1b1, 0x3, 0x2, 0x2, 0x2, 0x1b1, 0x1b2, 0x5, 0xba, 0x5e, 0x2, 0x1b2, 0x1b3, 0x7, - 0x83, 0x2, 0x2, 0x1b3, 0x1b4, 0x5, 0xe, 0x8, 0x2, 0x1b4, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x1b5, 0x1b6, 0x7, 0x6b, 0x2, 0x2, 0x1b6, 0x1b9, 0x7, - 0x1b, 0x2, 0x2, 0x1b7, 0x1b8, 0x7, 0x4c, 0x2, 0x2, 0x1b8, 0x1ba, 0x7, - 0x37, 0x2, 0x2, 0x1b9, 0x1b7, 0x3, 0x2, 0x2, 0x2, 0x1b9, 0x1ba, 0x3, + 0x84, 0x2, 0x2, 0x1b3, 0x1b4, 0x5, 0xe, 0x8, 0x2, 0x1b4, 0x1e5, 0x3, + 0x2, 0x2, 0x2, 0x1b5, 0x1b6, 0x7, 0x6c, 0x2, 0x2, 0x1b6, 0x1b9, 0x7, + 0x1c, 0x2, 0x2, 0x1b7, 0x1b8, 0x7, 0x4d, 0x2, 0x2, 0x1b8, 0x1ba, 0x7, + 0x38, 0x2, 0x2, 0x1b9, 0x1b7, 0x3, 0x2, 0x2, 0x2, 0x1b9, 0x1ba, 0x3, 0x2, 0x2, 0x2, 0x1ba, 0x1bb, 0x3, 0x2, 0x2, 0x2, 0x1bb, 0x1e5, 0x5, - 0x44, 0x23, 0x2, 0x1bc, 0x1bd, 0x7, 0x6b, 0x2, 0x2, 0x1bd, 0x1be, 0x7, - 0x78, 0x2, 0x2, 0x1be, 0x1bf, 0x7, 0x13, 0x2, 0x2, 0x1bf, 0x1e5, 0x5, - 0xb0, 0x59, 0x2, 0x1c0, 0x1c1, 0x7, 0x6b, 0x2, 0x2, 0x1c1, 0x1e5, 0x5, - 0x3e, 0x20, 0x2, 0x1c2, 0x1c3, 0x7, 0x6d, 0x2, 0x2, 0x1c3, 0x1cd, 0x5, - 0x10, 0x9, 0x2, 0x1c4, 0x1c5, 0x7, 0xa1, 0x2, 0x2, 0x1c5, 0x1c6, 0x7, - 0x2f, 0x2, 0x2, 0x1c6, 0x1ce, 0x7, 0xbe, 0x2, 0x2, 0x1c7, 0x1c8, 0x7, - 0xa1, 0x2, 0x2, 0x1c8, 0x1c9, 0x7, 0xb0, 0x2, 0x2, 0x1c9, 0x1ce, 0x7, - 0xbe, 0x2, 0x2, 0x1ca, 0x1cb, 0x7, 0xa1, 0x2, 0x2, 0x1cb, 0x1cc, 0x7, - 0x99, 0x2, 0x2, 0x1cc, 0x1ce, 0x5, 0xc0, 0x61, 0x2, 0x1cd, 0x1c4, 0x3, + 0x44, 0x23, 0x2, 0x1bc, 0x1bd, 0x7, 0x6c, 0x2, 0x2, 0x1bd, 0x1be, 0x7, + 0x79, 0x2, 0x2, 0x1be, 0x1bf, 0x7, 0x14, 0x2, 0x2, 0x1bf, 0x1e5, 0x5, + 0xb0, 0x59, 0x2, 0x1c0, 0x1c1, 0x7, 0x6c, 0x2, 0x2, 0x1c1, 0x1e5, 0x5, + 0x3e, 0x20, 0x2, 0x1c2, 0x1c3, 0x7, 0x6e, 0x2, 0x2, 0x1c3, 0x1cd, 0x5, + 0x10, 0x9, 0x2, 0x1c4, 0x1c5, 0x7, 0xa2, 0x2, 0x2, 0x1c5, 0x1c6, 0x7, + 0x30, 0x2, 0x2, 0x1c6, 0x1ce, 0x7, 0xbf, 0x2, 0x2, 0x1c7, 0x1c8, 0x7, + 0xa2, 0x2, 0x2, 0x1c8, 0x1c9, 0x7, 0xb1, 0x2, 0x2, 0x1c9, 0x1ce, 0x7, + 0xbf, 0x2, 0x2, 0x1ca, 0x1cb, 0x7, 0xa2, 0x2, 0x2, 0x1cb, 0x1cc, 0x7, + 0x9a, 0x2, 0x2, 0x1cc, 0x1ce, 0x5, 0xc0, 0x61, 0x2, 0x1cd, 0x1c4, 0x3, 0x2, 0x2, 0x2, 0x1cd, 0x1c7, 0x3, 0x2, 0x2, 0x2, 0x1cd, 0x1ca, 0x3, 0x2, 0x2, 0x2, 0x1ce, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x1cf, 0x1d0, 0x7, - 0x83, 0x2, 0x2, 0x1d0, 0x1e5, 0x7, 0xa7, 0x2, 0x2, 0x1d1, 0x1d2, 0x7, - 0x84, 0x2, 0x2, 0x1d2, 0x1d5, 0x7, 0x1b, 0x2, 0x2, 0x1d3, 0x1d4, 0x7, - 0x4c, 0x2, 0x2, 0x1d4, 0x1d6, 0x7, 0x37, 0x2, 0x2, 0x1d5, 0x1d3, 0x3, + 0x84, 0x2, 0x2, 0x1d0, 0x1e5, 0x7, 0xa8, 0x2, 0x2, 0x1d1, 0x1d2, 0x7, + 0x85, 0x2, 0x2, 0x1d2, 0x1d5, 0x7, 0x1c, 0x2, 0x2, 0x1d3, 0x1d4, 0x7, + 0x4d, 0x2, 0x2, 0x1d4, 0x1d6, 0x7, 0x38, 0x2, 0x2, 0x1d5, 0x1d3, 0x3, 0x2, 0x2, 0x2, 0x1d5, 0x1d6, 0x3, 0x2, 0x2, 0x2, 0x1d6, 0x1d7, 0x3, 0x2, 0x2, 0x2, 0x1d7, 0x1d8, 0x5, 0xba, 0x5e, 0x2, 0x1d8, 0x1d9, 0x7, - 0xa1, 0x2, 0x2, 0x1d9, 0x1da, 0x5, 0xba, 0x5e, 0x2, 0x1da, 0x1e5, 0x3, - 0x2, 0x2, 0x2, 0x1db, 0x1dc, 0x7, 0x85, 0x2, 0x2, 0x1dc, 0x1dd, 0x5, - 0x10, 0x9, 0x2, 0x1dd, 0x1de, 0x7, 0x42, 0x2, 0x2, 0x1de, 0x1df, 0x5, + 0xa2, 0x2, 0x2, 0x1d9, 0x1da, 0x5, 0xba, 0x5e, 0x2, 0x1da, 0x1e5, 0x3, + 0x2, 0x2, 0x2, 0x1db, 0x1dc, 0x7, 0x86, 0x2, 0x2, 0x1dc, 0x1dd, 0x5, + 0x10, 0x9, 0x2, 0x1dd, 0x1de, 0x7, 0x43, 0x2, 0x2, 0x1de, 0x1df, 0x5, 0xc0, 0x61, 0x2, 0x1df, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x1e0, 0x1e1, 0x7, - 0xaa, 0x2, 0x2, 0x1e1, 0x1e2, 0x5, 0xa, 0x6, 0x2, 0x1e2, 0x1e3, 0x5, + 0xab, 0x2, 0x2, 0x1e1, 0x1e2, 0x5, 0xa, 0x6, 0x2, 0x1e2, 0x1e3, 0x5, 0x78, 0x3d, 0x2, 0x1e3, 0x1e5, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x10e, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x11a, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x126, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x132, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x138, 0x3, @@ -19439,36 +19315,36 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x1e4, 0x1c2, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x1cf, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x1d1, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x1db, 0x3, 0x2, 0x2, 0x2, 0x1e4, 0x1e0, 0x3, 0x2, 0x2, 0x2, 0x1e5, 0x9, 0x3, 0x2, - 0x2, 0x2, 0x1e6, 0x1eb, 0x5, 0xc, 0x7, 0x2, 0x1e7, 0x1e8, 0x7, 0xc4, + 0x2, 0x2, 0x1e6, 0x1eb, 0x5, 0xc, 0x7, 0x2, 0x1e7, 0x1e8, 0x7, 0xc5, 0x2, 0x2, 0x1e8, 0x1ea, 0x5, 0xc, 0x7, 0x2, 0x1e9, 0x1e7, 0x3, 0x2, 0x2, 0x2, 0x1ea, 0x1ed, 0x3, 0x2, 0x2, 0x2, 0x1eb, 0x1e9, 0x3, 0x2, 0x2, 0x2, 0x1eb, 0x1ec, 0x3, 0x2, 0x2, 0x2, 0x1ec, 0xb, 0x3, 0x2, 0x2, 0x2, 0x1ed, 0x1eb, 0x3, 0x2, 0x2, 0x2, 0x1ee, 0x1ef, 0x5, 0xba, 0x5e, - 0x2, 0x1ef, 0x1f0, 0x7, 0xc9, 0x2, 0x2, 0x1f0, 0x1f1, 0x5, 0xb0, 0x59, + 0x2, 0x1ef, 0x1f0, 0x7, 0xca, 0x2, 0x2, 0x1f0, 0x1f1, 0x5, 0xb0, 0x59, 0x2, 0x1f1, 0xd, 0x3, 0x2, 0x2, 0x2, 0x1f2, 0x1f3, 0x9, 0x2, 0x2, 0x2, - 0x1f3, 0xf, 0x3, 0x2, 0x2, 0x2, 0x1f4, 0x1f5, 0x7, 0x7b, 0x2, 0x2, 0x1f5, - 0x1fa, 0x5, 0xb0, 0x59, 0x2, 0x1f6, 0x1f7, 0x7, 0x7b, 0x2, 0x2, 0x1f7, - 0x1f8, 0x7, 0x4b, 0x2, 0x2, 0x1f8, 0x1fa, 0x7, 0xbe, 0x2, 0x2, 0x1f9, + 0x1f3, 0xf, 0x3, 0x2, 0x2, 0x2, 0x1f4, 0x1f5, 0x7, 0x7c, 0x2, 0x2, 0x1f5, + 0x1fa, 0x5, 0xb0, 0x59, 0x2, 0x1f6, 0x1f7, 0x7, 0x7c, 0x2, 0x2, 0x1f7, + 0x1f8, 0x7, 0x4c, 0x2, 0x2, 0x1f8, 0x1fa, 0x7, 0xbf, 0x2, 0x2, 0x1f9, 0x1f4, 0x3, 0x2, 0x2, 0x2, 0x1f9, 0x1f6, 0x3, 0x2, 0x2, 0x2, 0x1fa, - 0x11, 0x3, 0x2, 0x2, 0x2, 0x1fb, 0x1fc, 0x7, 0x10, 0x2, 0x2, 0x1fc, - 0x1fd, 0x7, 0x2e, 0x2, 0x2, 0x1fd, 0x1ff, 0x5, 0xc0, 0x61, 0x2, 0x1fe, + 0x11, 0x3, 0x2, 0x2, 0x2, 0x1fb, 0x1fc, 0x7, 0x11, 0x2, 0x2, 0x1fc, + 0x1fd, 0x7, 0x2f, 0x2, 0x2, 0x1fd, 0x1ff, 0x5, 0xc0, 0x61, 0x2, 0x1fe, 0x200, 0x5, 0x2c, 0x17, 0x2, 0x1ff, 0x1fe, 0x3, 0x2, 0x2, 0x2, 0x1ff, 0x200, 0x3, 0x2, 0x2, 0x2, 0x200, 0x13, 0x3, 0x2, 0x2, 0x2, 0x201, 0x202, - 0x7, 0x16, 0x2, 0x2, 0x202, 0x203, 0x7, 0x99, 0x2, 0x2, 0x203, 0x205, + 0x7, 0x17, 0x2, 0x2, 0x202, 0x203, 0x7, 0x9a, 0x2, 0x2, 0x203, 0x205, 0x5, 0xc0, 0x61, 0x2, 0x204, 0x206, 0x5, 0x10, 0x9, 0x2, 0x205, 0x204, 0x3, 0x2, 0x2, 0x2, 0x205, 0x206, 0x3, 0x2, 0x2, 0x2, 0x206, 0x15, 0x3, 0x2, 0x2, 0x2, 0x207, 0x208, 0x9, 0x3, 0x2, 0x2, 0x208, 0x20c, 0x7, - 0x21, 0x2, 0x2, 0x209, 0x20a, 0x7, 0x4c, 0x2, 0x2, 0x20a, 0x20b, 0x7, - 0x71, 0x2, 0x2, 0x20b, 0x20d, 0x7, 0x37, 0x2, 0x2, 0x20c, 0x209, 0x3, + 0x22, 0x2, 0x2, 0x209, 0x20a, 0x7, 0x4d, 0x2, 0x2, 0x20a, 0x20b, 0x7, + 0x72, 0x2, 0x2, 0x20b, 0x20d, 0x7, 0x38, 0x2, 0x2, 0x20c, 0x209, 0x3, 0x2, 0x2, 0x2, 0x20c, 0x20d, 0x3, 0x2, 0x2, 0x2, 0x20d, 0x20e, 0x3, 0x2, 0x2, 0x2, 0x20e, 0x210, 0x5, 0xc6, 0x64, 0x2, 0x20f, 0x211, 0x5, 0x2c, 0x17, 0x2, 0x210, 0x20f, 0x3, 0x2, 0x2, 0x2, 0x210, 0x211, 0x3, 0x2, 0x2, 0x2, 0x211, 0x213, 0x3, 0x2, 0x2, 0x2, 0x212, 0x214, 0x5, 0x40, 0x21, 0x2, 0x213, 0x212, 0x3, 0x2, 0x2, 0x2, 0x213, 0x214, 0x3, 0x2, 0x2, 0x2, 0x214, 0x291, 0x3, 0x2, 0x2, 0x2, 0x215, 0x216, 0x9, - 0x3, 0x2, 0x2, 0x216, 0x21a, 0x7, 0x2e, 0x2, 0x2, 0x217, 0x218, 0x7, - 0x4c, 0x2, 0x2, 0x218, 0x219, 0x7, 0x71, 0x2, 0x2, 0x219, 0x21b, 0x7, - 0x37, 0x2, 0x2, 0x21a, 0x217, 0x3, 0x2, 0x2, 0x2, 0x21a, 0x21b, 0x3, + 0x3, 0x2, 0x2, 0x216, 0x21a, 0x7, 0x2f, 0x2, 0x2, 0x217, 0x218, 0x7, + 0x4d, 0x2, 0x2, 0x218, 0x219, 0x7, 0x72, 0x2, 0x2, 0x219, 0x21b, 0x7, + 0x38, 0x2, 0x2, 0x21a, 0x217, 0x3, 0x2, 0x2, 0x2, 0x21a, 0x21b, 0x3, 0x2, 0x2, 0x2, 0x21b, 0x21c, 0x3, 0x2, 0x2, 0x2, 0x21c, 0x21e, 0x5, 0xc0, 0x61, 0x2, 0x21d, 0x21f, 0x5, 0x2e, 0x18, 0x2, 0x21e, 0x21d, 0x3, 0x2, 0x2, 0x2, 0x21e, 0x21f, 0x3, 0x2, 0x2, 0x2, 0x21f, 0x221, 0x3, @@ -19476,17 +19352,17 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x221, 0x222, 0x3, 0x2, 0x2, 0x2, 0x222, 0x223, 0x3, 0x2, 0x2, 0x2, 0x223, 0x224, 0x5, 0x18, 0xd, 0x2, 0x224, 0x225, 0x5, 0x1c, 0xf, 0x2, 0x225, 0x291, 0x3, 0x2, 0x2, 0x2, 0x226, 0x227, 0x9, - 0x3, 0x2, 0x2, 0x227, 0x228, 0x7, 0x62, 0x2, 0x2, 0x228, 0x22c, 0x7, - 0xaf, 0x2, 0x2, 0x229, 0x22a, 0x7, 0x4c, 0x2, 0x2, 0x22a, 0x22b, 0x7, - 0x71, 0x2, 0x2, 0x22b, 0x22d, 0x7, 0x37, 0x2, 0x2, 0x22c, 0x229, 0x3, + 0x3, 0x2, 0x2, 0x227, 0x228, 0x7, 0x63, 0x2, 0x2, 0x228, 0x22c, 0x7, + 0xb0, 0x2, 0x2, 0x229, 0x22a, 0x7, 0x4d, 0x2, 0x2, 0x22a, 0x22b, 0x7, + 0x72, 0x2, 0x2, 0x22b, 0x22d, 0x7, 0x38, 0x2, 0x2, 0x22c, 0x229, 0x3, 0x2, 0x2, 0x2, 0x22c, 0x22d, 0x3, 0x2, 0x2, 0x2, 0x22d, 0x22e, 0x3, 0x2, 0x2, 0x2, 0x22e, 0x230, 0x5, 0xc0, 0x61, 0x2, 0x22f, 0x231, 0x5, 0x2e, 0x18, 0x2, 0x230, 0x22f, 0x3, 0x2, 0x2, 0x2, 0x230, 0x231, 0x3, 0x2, 0x2, 0x2, 0x231, 0x233, 0x3, 0x2, 0x2, 0x2, 0x232, 0x234, 0x5, 0x2c, 0x17, 0x2, 0x233, 0x232, 0x3, 0x2, 0x2, 0x2, 0x233, 0x234, 0x3, 0x2, 0x2, 0x2, 0x234, 0x23a, 0x3, 0x2, 0x2, 0x2, 0x235, 0x236, 0x7, - 0xb5, 0x2, 0x2, 0x236, 0x238, 0x7, 0x9f, 0x2, 0x2, 0x237, 0x239, 0x7, - 0xbc, 0x2, 0x2, 0x238, 0x237, 0x3, 0x2, 0x2, 0x2, 0x238, 0x239, 0x3, + 0xb6, 0x2, 0x2, 0x236, 0x238, 0x7, 0xa0, 0x2, 0x2, 0x237, 0x239, 0x7, + 0xbd, 0x2, 0x2, 0x238, 0x237, 0x3, 0x2, 0x2, 0x2, 0x238, 0x239, 0x3, 0x2, 0x2, 0x2, 0x239, 0x23b, 0x3, 0x2, 0x2, 0x2, 0x23a, 0x235, 0x3, 0x2, 0x2, 0x2, 0x23a, 0x23b, 0x3, 0x2, 0x2, 0x2, 0x23b, 0x23d, 0x3, 0x2, 0x2, 0x2, 0x23c, 0x23e, 0x5, 0x30, 0x19, 0x2, 0x23d, 0x23c, 0x3, @@ -19495,9 +19371,9 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x240, 0x241, 0x3, 0x2, 0x2, 0x2, 0x241, 0x242, 0x3, 0x2, 0x2, 0x2, 0x242, 0x243, 0x5, 0x32, 0x1a, 0x2, 0x243, 0x291, 0x3, 0x2, 0x2, 0x2, 0x244, 0x245, 0x9, 0x3, 0x2, 0x2, 0x245, 0x246, 0x7, - 0x65, 0x2, 0x2, 0x246, 0x24a, 0x7, 0xaf, 0x2, 0x2, 0x247, 0x248, 0x7, - 0x4c, 0x2, 0x2, 0x248, 0x249, 0x7, 0x71, 0x2, 0x2, 0x249, 0x24b, 0x7, - 0x37, 0x2, 0x2, 0x24a, 0x247, 0x3, 0x2, 0x2, 0x2, 0x24a, 0x24b, 0x3, + 0x67, 0x2, 0x2, 0x246, 0x24a, 0x7, 0xb0, 0x2, 0x2, 0x247, 0x248, 0x7, + 0x4d, 0x2, 0x2, 0x248, 0x249, 0x7, 0x72, 0x2, 0x2, 0x249, 0x24b, 0x7, + 0x38, 0x2, 0x2, 0x24a, 0x247, 0x3, 0x2, 0x2, 0x2, 0x24a, 0x24b, 0x3, 0x2, 0x2, 0x2, 0x24b, 0x24c, 0x3, 0x2, 0x2, 0x2, 0x24c, 0x24e, 0x5, 0xc0, 0x61, 0x2, 0x24d, 0x24f, 0x5, 0x2e, 0x18, 0x2, 0x24e, 0x24d, 0x3, 0x2, 0x2, 0x2, 0x24e, 0x24f, 0x3, 0x2, 0x2, 0x2, 0x24f, 0x251, 0x3, @@ -19506,16 +19382,16 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x253, 0x255, 0x5, 0x34, 0x1b, 0x2, 0x254, 0x253, 0x3, 0x2, 0x2, 0x2, 0x254, 0x255, 0x3, 0x2, 0x2, 0x2, 0x255, 0x25b, 0x3, 0x2, 0x2, 0x2, 0x256, 0x25c, 0x5, 0x30, 0x19, 0x2, 0x257, 0x259, 0x5, - 0x36, 0x1c, 0x2, 0x258, 0x25a, 0x7, 0x7c, 0x2, 0x2, 0x259, 0x258, 0x3, + 0x36, 0x1c, 0x2, 0x258, 0x25a, 0x7, 0x7d, 0x2, 0x2, 0x259, 0x258, 0x3, 0x2, 0x2, 0x2, 0x259, 0x25a, 0x3, 0x2, 0x2, 0x2, 0x25a, 0x25c, 0x3, 0x2, 0x2, 0x2, 0x25b, 0x256, 0x3, 0x2, 0x2, 0x2, 0x25b, 0x257, 0x3, 0x2, 0x2, 0x2, 0x25c, 0x25d, 0x3, 0x2, 0x2, 0x2, 0x25d, 0x25e, 0x5, 0x32, 0x1a, 0x2, 0x25e, 0x291, 0x3, 0x2, 0x2, 0x2, 0x25f, 0x261, 0x9, - 0x3, 0x2, 0x2, 0x260, 0x262, 0x7, 0x9b, 0x2, 0x2, 0x261, 0x260, 0x3, + 0x3, 0x2, 0x2, 0x260, 0x262, 0x7, 0x9c, 0x2, 0x2, 0x261, 0x260, 0x3, 0x2, 0x2, 0x2, 0x261, 0x262, 0x3, 0x2, 0x2, 0x2, 0x262, 0x263, 0x3, - 0x2, 0x2, 0x2, 0x263, 0x267, 0x7, 0x99, 0x2, 0x2, 0x264, 0x265, 0x7, - 0x4c, 0x2, 0x2, 0x265, 0x266, 0x7, 0x71, 0x2, 0x2, 0x266, 0x268, 0x7, - 0x37, 0x2, 0x2, 0x267, 0x264, 0x3, 0x2, 0x2, 0x2, 0x267, 0x268, 0x3, + 0x2, 0x2, 0x2, 0x263, 0x267, 0x7, 0x9a, 0x2, 0x2, 0x264, 0x265, 0x7, + 0x4d, 0x2, 0x2, 0x265, 0x266, 0x7, 0x72, 0x2, 0x2, 0x266, 0x268, 0x7, + 0x38, 0x2, 0x2, 0x267, 0x264, 0x3, 0x2, 0x2, 0x2, 0x267, 0x268, 0x3, 0x2, 0x2, 0x2, 0x268, 0x269, 0x3, 0x2, 0x2, 0x2, 0x269, 0x26b, 0x5, 0xc0, 0x61, 0x2, 0x26a, 0x26c, 0x5, 0x2e, 0x18, 0x2, 0x26b, 0x26a, 0x3, 0x2, 0x2, 0x2, 0x26b, 0x26c, 0x3, 0x2, 0x2, 0x2, 0x26c, 0x26e, 0x3, @@ -19528,11 +19404,11 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x276, 0x278, 0x5, 0x32, 0x1a, 0x2, 0x277, 0x276, 0x3, 0x2, 0x2, 0x2, 0x277, 0x278, 0x3, 0x2, 0x2, 0x2, 0x278, 0x291, 0x3, 0x2, 0x2, 0x2, 0x279, 0x27c, 0x9, 0x3, 0x2, 0x2, 0x27a, 0x27b, 0x7, - 0x77, 0x2, 0x2, 0x27b, 0x27d, 0x7, 0x85, 0x2, 0x2, 0x27c, 0x27a, 0x3, + 0x78, 0x2, 0x2, 0x27b, 0x27d, 0x7, 0x86, 0x2, 0x2, 0x27c, 0x27a, 0x3, 0x2, 0x2, 0x2, 0x27c, 0x27d, 0x3, 0x2, 0x2, 0x2, 0x27d, 0x27e, 0x3, - 0x2, 0x2, 0x2, 0x27e, 0x282, 0x7, 0xaf, 0x2, 0x2, 0x27f, 0x280, 0x7, - 0x4c, 0x2, 0x2, 0x280, 0x281, 0x7, 0x71, 0x2, 0x2, 0x281, 0x283, 0x7, - 0x37, 0x2, 0x2, 0x282, 0x27f, 0x3, 0x2, 0x2, 0x2, 0x282, 0x283, 0x3, + 0x2, 0x2, 0x2, 0x27e, 0x282, 0x7, 0xb0, 0x2, 0x2, 0x27f, 0x280, 0x7, + 0x4d, 0x2, 0x2, 0x280, 0x281, 0x7, 0x72, 0x2, 0x2, 0x281, 0x283, 0x7, + 0x38, 0x2, 0x2, 0x282, 0x27f, 0x3, 0x2, 0x2, 0x2, 0x282, 0x283, 0x3, 0x2, 0x2, 0x2, 0x283, 0x284, 0x3, 0x2, 0x2, 0x2, 0x284, 0x286, 0x5, 0xc0, 0x61, 0x2, 0x285, 0x287, 0x5, 0x2e, 0x18, 0x2, 0x286, 0x285, 0x3, 0x2, 0x2, 0x2, 0x286, 0x287, 0x3, 0x2, 0x2, 0x2, 0x287, 0x289, 0x3, @@ -19544,23 +19420,23 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2, 0x2, 0x290, 0x207, 0x3, 0x2, 0x2, 0x2, 0x290, 0x215, 0x3, 0x2, 0x2, 0x2, 0x290, 0x226, 0x3, 0x2, 0x2, 0x2, 0x290, 0x244, 0x3, 0x2, 0x2, 0x2, 0x290, 0x25f, 0x3, 0x2, 0x2, 0x2, 0x290, 0x279, 0x3, - 0x2, 0x2, 0x2, 0x291, 0x17, 0x3, 0x2, 0x2, 0x2, 0x292, 0x293, 0x7, 0xcf, - 0x2, 0x2, 0x293, 0x298, 0x5, 0x1a, 0xe, 0x2, 0x294, 0x295, 0x7, 0xc4, + 0x2, 0x2, 0x2, 0x291, 0x17, 0x3, 0x2, 0x2, 0x2, 0x292, 0x293, 0x7, 0xd0, + 0x2, 0x2, 0x293, 0x298, 0x5, 0x1a, 0xe, 0x2, 0x294, 0x295, 0x7, 0xc5, 0x2, 0x2, 0x295, 0x297, 0x5, 0x1a, 0xe, 0x2, 0x296, 0x294, 0x3, 0x2, 0x2, 0x2, 0x297, 0x29a, 0x3, 0x2, 0x2, 0x2, 0x298, 0x296, 0x3, 0x2, 0x2, 0x2, 0x298, 0x299, 0x3, 0x2, 0x2, 0x2, 0x299, 0x29b, 0x3, 0x2, - 0x2, 0x2, 0x29a, 0x298, 0x3, 0x2, 0x2, 0x2, 0x29b, 0x29c, 0x7, 0xd9, + 0x2, 0x2, 0x29a, 0x298, 0x3, 0x2, 0x2, 0x2, 0x29b, 0x29c, 0x7, 0xda, 0x2, 0x2, 0x29c, 0x19, 0x3, 0x2, 0x2, 0x2, 0x29d, 0x29e, 0x5, 0xd6, 0x6c, 0x2, 0x29e, 0x2b4, 0x5, 0xaa, 0x56, 0x2, 0x29f, 0x2a0, 0x6, 0xe, - 0x2, 0x3, 0x2a0, 0x2a1, 0x7, 0x26, 0x2, 0x2, 0x2a1, 0x2a2, 0x5, 0xcc, + 0x2, 0x3, 0x2a0, 0x2a1, 0x7, 0x27, 0x2, 0x2, 0x2a1, 0x2a2, 0x5, 0xcc, 0x67, 0x2, 0x2a2, 0x2a3, 0x8, 0xe, 0x1, 0x2, 0x2a3, 0x2b3, 0x3, 0x2, - 0x2, 0x2, 0x2a4, 0x2a5, 0x6, 0xe, 0x3, 0x3, 0x2a5, 0x2a6, 0x7, 0x39, + 0x2, 0x2, 0x2a4, 0x2a5, 0x6, 0xe, 0x3, 0x3, 0x2a5, 0x2a6, 0x7, 0x3a, 0x2, 0x2, 0x2a6, 0x2a7, 0x5, 0xb0, 0x59, 0x2, 0x2a7, 0x2a8, 0x8, 0xe, 0x1, 0x2, 0x2a8, 0x2b3, 0x3, 0x2, 0x2, 0x2, 0x2a9, 0x2aa, 0x6, 0xe, - 0x4, 0x3, 0x2aa, 0x2ab, 0x7, 0x49, 0x2, 0x2, 0x2ab, 0x2b3, 0x8, 0xe, - 0x1, 0x2, 0x2ac, 0x2ad, 0x6, 0xe, 0x5, 0x3, 0x2ad, 0x2ae, 0x7, 0x51, + 0x4, 0x3, 0x2aa, 0x2ab, 0x7, 0x4a, 0x2, 0x2, 0x2ab, 0x2b3, 0x8, 0xe, + 0x1, 0x2, 0x2ac, 0x2ad, 0x6, 0xe, 0x5, 0x3, 0x2ad, 0x2ae, 0x7, 0x52, 0x2, 0x2, 0x2ae, 0x2b3, 0x8, 0xe, 0x1, 0x2, 0x2af, 0x2b0, 0x6, 0xe, - 0x6, 0x3, 0x2b0, 0x2b1, 0x7, 0x57, 0x2, 0x2, 0x2b1, 0x2b3, 0x8, 0xe, + 0x6, 0x3, 0x2b0, 0x2b1, 0x7, 0x58, 0x2, 0x2, 0x2b1, 0x2b3, 0x8, 0xe, 0x1, 0x2, 0x2b2, 0x29f, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2a4, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2a9, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2ac, 0x3, 0x2, 0x2, 0x2, 0x2b2, 0x2af, 0x3, 0x2, 0x2, 0x2, 0x2b3, 0x2b6, 0x3, 0x2, @@ -19583,61 +19459,61 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x2ce, 0x2ca, 0x3, 0x2, 0x2, 0x2, 0x2cf, 0x2d2, 0x3, 0x2, 0x2, 0x2, 0x2d0, 0x2ce, 0x3, 0x2, 0x2, 0x2, 0x2d0, 0x2d1, 0x3, 0x2, 0x2, 0x2, 0x2d1, 0x1d, 0x3, 0x2, 0x2, 0x2, 0x2d2, 0x2d0, 0x3, 0x2, 0x2, 0x2, - 0x2d3, 0x2d4, 0x7, 0x7e, 0x2, 0x2, 0x2d4, 0x2d5, 0x7, 0x59, 0x2, 0x2, + 0x2d3, 0x2d4, 0x7, 0x7f, 0x2, 0x2, 0x2d4, 0x2d5, 0x7, 0x5a, 0x2, 0x2, 0x2d5, 0x2d6, 0x5, 0xac, 0x57, 0x2, 0x2d6, 0x1f, 0x3, 0x2, 0x2, 0x2, 0x2d7, 0x2de, 0x5, 0xd6, 0x6c, 0x2, 0x2d8, 0x2db, 0x5, 0xd6, 0x6c, 0x2, - 0x2d9, 0x2da, 0x7, 0xcf, 0x2, 0x2, 0x2da, 0x2dc, 0x7, 0xd9, 0x2, 0x2, + 0x2d9, 0x2da, 0x7, 0xd0, 0x2, 0x2, 0x2da, 0x2dc, 0x7, 0xda, 0x2, 0x2, 0x2db, 0x2d9, 0x3, 0x2, 0x2, 0x2, 0x2db, 0x2dc, 0x3, 0x2, 0x2, 0x2, 0x2dc, 0x2df, 0x3, 0x2, 0x2, 0x2, 0x2dd, 0x2df, 0x5, 0xcc, 0x67, 0x2, 0x2de, 0x2d8, 0x3, 0x2, 0x2, 0x2, 0x2de, 0x2dd, 0x3, 0x2, 0x2, 0x2, - 0x2df, 0x21, 0x3, 0x2, 0x2, 0x2, 0x2e0, 0x2e1, 0x7, 0x92, 0x2, 0x2, - 0x2e1, 0x2e2, 0x7, 0xcf, 0x2, 0x2, 0x2e2, 0x2e3, 0x5, 0xd6, 0x6c, 0x2, - 0x2e3, 0x2e7, 0x7, 0xcf, 0x2, 0x2, 0x2e4, 0x2e6, 0x5, 0x20, 0x11, 0x2, + 0x2df, 0x21, 0x3, 0x2, 0x2, 0x2, 0x2e0, 0x2e1, 0x7, 0x93, 0x2, 0x2, + 0x2e1, 0x2e2, 0x7, 0xd0, 0x2, 0x2, 0x2e2, 0x2e3, 0x5, 0xd6, 0x6c, 0x2, + 0x2e3, 0x2e7, 0x7, 0xd0, 0x2, 0x2, 0x2e4, 0x2e6, 0x5, 0x20, 0x11, 0x2, 0x2e5, 0x2e4, 0x3, 0x2, 0x2, 0x2, 0x2e6, 0x2e9, 0x3, 0x2, 0x2, 0x2, 0x2e7, 0x2e5, 0x3, 0x2, 0x2, 0x2, 0x2e7, 0x2e8, 0x3, 0x2, 0x2, 0x2, 0x2e8, 0x2ea, 0x3, 0x2, 0x2, 0x2, 0x2e9, 0x2e7, 0x3, 0x2, 0x2, 0x2, - 0x2ea, 0x2eb, 0x7, 0xd9, 0x2, 0x2, 0x2eb, 0x2ec, 0x7, 0xd9, 0x2, 0x2, - 0x2ec, 0x23, 0x3, 0x2, 0x2, 0x2, 0x2ed, 0x2ee, 0x7, 0x5f, 0x2, 0x2, - 0x2ee, 0x2f8, 0x7, 0xcf, 0x2, 0x2, 0x2ef, 0x2f9, 0x7, 0xbc, 0x2, 0x2, - 0x2f0, 0x2f1, 0x7, 0x69, 0x2, 0x2, 0x2f1, 0x2f2, 0x7, 0xbc, 0x2, 0x2, - 0x2f2, 0x2f3, 0x7, 0x67, 0x2, 0x2, 0x2f3, 0x2f9, 0x7, 0xbc, 0x2, 0x2, - 0x2f4, 0x2f5, 0x7, 0x67, 0x2, 0x2, 0x2f5, 0x2f6, 0x7, 0xbc, 0x2, 0x2, - 0x2f6, 0x2f7, 0x7, 0x69, 0x2, 0x2, 0x2f7, 0x2f9, 0x7, 0xbc, 0x2, 0x2, + 0x2ea, 0x2eb, 0x7, 0xda, 0x2, 0x2, 0x2eb, 0x2ec, 0x7, 0xda, 0x2, 0x2, + 0x2ec, 0x23, 0x3, 0x2, 0x2, 0x2, 0x2ed, 0x2ee, 0x7, 0x60, 0x2, 0x2, + 0x2ee, 0x2f8, 0x7, 0xd0, 0x2, 0x2, 0x2ef, 0x2f9, 0x7, 0xbd, 0x2, 0x2, + 0x2f0, 0x2f1, 0x7, 0x6a, 0x2, 0x2, 0x2f1, 0x2f2, 0x7, 0xbd, 0x2, 0x2, + 0x2f2, 0x2f3, 0x7, 0x68, 0x2, 0x2, 0x2f3, 0x2f9, 0x7, 0xbd, 0x2, 0x2, + 0x2f4, 0x2f5, 0x7, 0x68, 0x2, 0x2, 0x2f5, 0x2f6, 0x7, 0xbd, 0x2, 0x2, + 0x2f6, 0x2f7, 0x7, 0x6a, 0x2, 0x2, 0x2f7, 0x2f9, 0x7, 0xbd, 0x2, 0x2, 0x2f8, 0x2ef, 0x3, 0x2, 0x2, 0x2, 0x2f8, 0x2f0, 0x3, 0x2, 0x2, 0x2, 0x2f8, 0x2f4, 0x3, 0x2, 0x2, 0x2, 0x2f9, 0x2fa, 0x3, 0x2, 0x2, 0x2, - 0x2fa, 0x2fb, 0x7, 0xd9, 0x2, 0x2, 0x2fb, 0x25, 0x3, 0x2, 0x2, 0x2, - 0x2fc, 0x2fd, 0x7, 0x5c, 0x2, 0x2, 0x2fd, 0x2fe, 0x7, 0xcf, 0x2, 0x2, - 0x2fe, 0x2ff, 0x5, 0xd6, 0x6c, 0x2, 0x2ff, 0x303, 0x7, 0xcf, 0x2, 0x2, + 0x2fa, 0x2fb, 0x7, 0xda, 0x2, 0x2, 0x2fb, 0x25, 0x3, 0x2, 0x2, 0x2, + 0x2fc, 0x2fd, 0x7, 0x5d, 0x2, 0x2, 0x2fd, 0x2fe, 0x7, 0xd0, 0x2, 0x2, + 0x2fe, 0x2ff, 0x5, 0xd6, 0x6c, 0x2, 0x2ff, 0x303, 0x7, 0xd0, 0x2, 0x2, 0x300, 0x302, 0x5, 0x20, 0x11, 0x2, 0x301, 0x300, 0x3, 0x2, 0x2, 0x2, 0x302, 0x305, 0x3, 0x2, 0x2, 0x2, 0x303, 0x301, 0x3, 0x2, 0x2, 0x2, 0x303, 0x304, 0x3, 0x2, 0x2, 0x2, 0x304, 0x306, 0x3, 0x2, 0x2, 0x2, - 0x305, 0x303, 0x3, 0x2, 0x2, 0x2, 0x306, 0x307, 0x7, 0xd9, 0x2, 0x2, - 0x307, 0x308, 0x7, 0xd9, 0x2, 0x2, 0x308, 0x27, 0x3, 0x2, 0x2, 0x2, - 0x309, 0x30a, 0x7, 0x81, 0x2, 0x2, 0x30a, 0x315, 0x7, 0xcf, 0x2, 0x2, - 0x30b, 0x30c, 0x7, 0x69, 0x2, 0x2, 0x30c, 0x30d, 0x5, 0xd6, 0x6c, 0x2, - 0x30d, 0x30e, 0x7, 0x67, 0x2, 0x2, 0x30e, 0x30f, 0x5, 0xd6, 0x6c, 0x2, - 0x30f, 0x316, 0x3, 0x2, 0x2, 0x2, 0x310, 0x311, 0x7, 0x67, 0x2, 0x2, - 0x311, 0x312, 0x5, 0xd6, 0x6c, 0x2, 0x312, 0x313, 0x7, 0x69, 0x2, 0x2, + 0x305, 0x303, 0x3, 0x2, 0x2, 0x2, 0x306, 0x307, 0x7, 0xda, 0x2, 0x2, + 0x307, 0x308, 0x7, 0xda, 0x2, 0x2, 0x308, 0x27, 0x3, 0x2, 0x2, 0x2, + 0x309, 0x30a, 0x7, 0x82, 0x2, 0x2, 0x30a, 0x315, 0x7, 0xd0, 0x2, 0x2, + 0x30b, 0x30c, 0x7, 0x6a, 0x2, 0x2, 0x30c, 0x30d, 0x5, 0xd6, 0x6c, 0x2, + 0x30d, 0x30e, 0x7, 0x68, 0x2, 0x2, 0x30e, 0x30f, 0x5, 0xd6, 0x6c, 0x2, + 0x30f, 0x316, 0x3, 0x2, 0x2, 0x2, 0x310, 0x311, 0x7, 0x68, 0x2, 0x2, + 0x311, 0x312, 0x5, 0xd6, 0x6c, 0x2, 0x312, 0x313, 0x7, 0x6a, 0x2, 0x2, 0x313, 0x314, 0x5, 0xd6, 0x6c, 0x2, 0x314, 0x316, 0x3, 0x2, 0x2, 0x2, 0x315, 0x30b, 0x3, 0x2, 0x2, 0x2, 0x315, 0x310, 0x3, 0x2, 0x2, 0x2, - 0x316, 0x317, 0x3, 0x2, 0x2, 0x2, 0x317, 0x318, 0x7, 0xd9, 0x2, 0x2, - 0x318, 0x29, 0x3, 0x2, 0x2, 0x2, 0x319, 0x31a, 0x7, 0x90, 0x2, 0x2, - 0x31a, 0x31b, 0x7, 0xcf, 0x2, 0x2, 0x31b, 0x31c, 0x5, 0x9a, 0x4e, 0x2, - 0x31c, 0x31d, 0x7, 0xd9, 0x2, 0x2, 0x31d, 0x2b, 0x3, 0x2, 0x2, 0x2, - 0x31e, 0x31f, 0x7, 0x75, 0x2, 0x2, 0x31f, 0x322, 0x7, 0x18, 0x2, 0x2, - 0x320, 0x323, 0x5, 0xd6, 0x6c, 0x2, 0x321, 0x323, 0x7, 0xbe, 0x2, 0x2, + 0x316, 0x317, 0x3, 0x2, 0x2, 0x2, 0x317, 0x318, 0x7, 0xda, 0x2, 0x2, + 0x318, 0x29, 0x3, 0x2, 0x2, 0x2, 0x319, 0x31a, 0x7, 0x91, 0x2, 0x2, + 0x31a, 0x31b, 0x7, 0xd0, 0x2, 0x2, 0x31b, 0x31c, 0x5, 0x9a, 0x4e, 0x2, + 0x31c, 0x31d, 0x7, 0xda, 0x2, 0x2, 0x31d, 0x2b, 0x3, 0x2, 0x2, 0x2, + 0x31e, 0x31f, 0x7, 0x76, 0x2, 0x2, 0x31f, 0x322, 0x7, 0x19, 0x2, 0x2, + 0x320, 0x323, 0x5, 0xd6, 0x6c, 0x2, 0x321, 0x323, 0x7, 0xbf, 0x2, 0x2, 0x322, 0x320, 0x3, 0x2, 0x2, 0x2, 0x322, 0x321, 0x3, 0x2, 0x2, 0x2, - 0x323, 0x2d, 0x3, 0x2, 0x2, 0x2, 0x324, 0x325, 0x7, 0xad, 0x2, 0x2, - 0x325, 0x326, 0x7, 0xbe, 0x2, 0x2, 0x326, 0x2f, 0x3, 0x2, 0x2, 0x2, - 0x327, 0x328, 0x7, 0xa1, 0x2, 0x2, 0x328, 0x329, 0x5, 0xc0, 0x61, 0x2, + 0x323, 0x2d, 0x3, 0x2, 0x2, 0x2, 0x324, 0x325, 0x7, 0xae, 0x2, 0x2, + 0x325, 0x326, 0x7, 0xbf, 0x2, 0x2, 0x326, 0x2f, 0x3, 0x2, 0x2, 0x2, + 0x327, 0x328, 0x7, 0xa2, 0x2, 0x2, 0x328, 0x329, 0x5, 0xc0, 0x61, 0x2, 0x329, 0x31, 0x3, 0x2, 0x2, 0x2, 0x32a, 0x32b, 0x7, 0xc, 0x2, 0x2, 0x32b, 0x32c, 0x5, 0x68, 0x35, 0x2, 0x32c, 0x33, 0x3, 0x2, 0x2, 0x2, 0x32d, - 0x32e, 0x7, 0xcf, 0x2, 0x2, 0x32e, 0x333, 0x5, 0x42, 0x22, 0x2, 0x32f, - 0x330, 0x7, 0xc4, 0x2, 0x2, 0x330, 0x332, 0x5, 0x42, 0x22, 0x2, 0x331, + 0x32e, 0x7, 0xd0, 0x2, 0x2, 0x32e, 0x333, 0x5, 0x42, 0x22, 0x2, 0x32f, + 0x330, 0x7, 0xc5, 0x2, 0x2, 0x330, 0x332, 0x5, 0x42, 0x22, 0x2, 0x331, 0x32f, 0x3, 0x2, 0x2, 0x2, 0x332, 0x335, 0x3, 0x2, 0x2, 0x2, 0x333, 0x331, 0x3, 0x2, 0x2, 0x2, 0x333, 0x334, 0x3, 0x2, 0x2, 0x2, 0x334, 0x336, 0x3, 0x2, 0x2, 0x2, 0x335, 0x333, 0x3, 0x2, 0x2, 0x2, 0x336, - 0x337, 0x7, 0xd9, 0x2, 0x2, 0x337, 0x33d, 0x3, 0x2, 0x2, 0x2, 0x338, + 0x337, 0x7, 0xda, 0x2, 0x2, 0x337, 0x33d, 0x3, 0x2, 0x2, 0x2, 0x338, 0x339, 0x7, 0xc, 0x2, 0x2, 0x339, 0x33d, 0x5, 0xc0, 0x61, 0x2, 0x33a, 0x33b, 0x7, 0xc, 0x2, 0x2, 0x33b, 0x33d, 0x5, 0xbe, 0x60, 0x2, 0x33c, 0x32d, 0x3, 0x2, 0x2, 0x2, 0x33c, 0x338, 0x3, 0x2, 0x2, 0x2, 0x33c, @@ -19660,672 +19536,675 @@ ClickHouseParser::Initializer::Initializer() { 0x3, 0x2, 0x2, 0x2, 0x358, 0x35b, 0x3, 0x2, 0x2, 0x2, 0x359, 0x357, 0x3, 0x2, 0x2, 0x2, 0x359, 0x35a, 0x3, 0x2, 0x2, 0x2, 0x35a, 0x37, 0x3, 0x2, 0x2, 0x2, 0x35b, 0x359, 0x3, 0x2, 0x2, 0x2, 0x35c, 0x35d, 0x7, - 0x7b, 0x2, 0x2, 0x35d, 0x35e, 0x7, 0x13, 0x2, 0x2, 0x35e, 0x35f, 0x5, + 0x7c, 0x2, 0x2, 0x35d, 0x35e, 0x7, 0x14, 0x2, 0x2, 0x35e, 0x35f, 0x5, 0xb0, 0x59, 0x2, 0x35f, 0x39, 0x3, 0x2, 0x2, 0x2, 0x360, 0x361, 0x7, - 0x7e, 0x2, 0x2, 0x361, 0x362, 0x7, 0x59, 0x2, 0x2, 0x362, 0x363, 0x5, + 0x7f, 0x2, 0x2, 0x361, 0x362, 0x7, 0x5a, 0x2, 0x2, 0x362, 0x363, 0x5, 0xb0, 0x59, 0x2, 0x363, 0x3b, 0x3, 0x2, 0x2, 0x2, 0x364, 0x365, 0x7, - 0x8a, 0x2, 0x2, 0x365, 0x366, 0x7, 0x13, 0x2, 0x2, 0x366, 0x367, 0x5, + 0x8b, 0x2, 0x2, 0x365, 0x366, 0x7, 0x14, 0x2, 0x2, 0x366, 0x367, 0x5, 0xb0, 0x59, 0x2, 0x367, 0x3d, 0x3, 0x2, 0x2, 0x2, 0x368, 0x369, 0x7, - 0xa7, 0x2, 0x2, 0x369, 0x36e, 0x5, 0x50, 0x29, 0x2, 0x36a, 0x36b, 0x7, - 0xc4, 0x2, 0x2, 0x36b, 0x36d, 0x5, 0x50, 0x29, 0x2, 0x36c, 0x36a, 0x3, + 0xa8, 0x2, 0x2, 0x369, 0x36e, 0x5, 0x50, 0x29, 0x2, 0x36a, 0x36b, 0x7, + 0xc5, 0x2, 0x2, 0x36b, 0x36d, 0x5, 0x50, 0x29, 0x2, 0x36c, 0x36a, 0x3, 0x2, 0x2, 0x2, 0x36d, 0x370, 0x3, 0x2, 0x2, 0x2, 0x36e, 0x36c, 0x3, 0x2, 0x2, 0x2, 0x36e, 0x36f, 0x3, 0x2, 0x2, 0x2, 0x36f, 0x3f, 0x3, 0x2, - 0x2, 0x2, 0x370, 0x36e, 0x3, 0x2, 0x2, 0x2, 0x371, 0x373, 0x7, 0x35, - 0x2, 0x2, 0x372, 0x374, 0x7, 0xc9, 0x2, 0x2, 0x373, 0x372, 0x3, 0x2, + 0x2, 0x2, 0x370, 0x36e, 0x3, 0x2, 0x2, 0x2, 0x371, 0x373, 0x7, 0x36, + 0x2, 0x2, 0x372, 0x374, 0x7, 0xca, 0x2, 0x2, 0x373, 0x372, 0x3, 0x2, 0x2, 0x2, 0x373, 0x374, 0x3, 0x2, 0x2, 0x2, 0x374, 0x375, 0x3, 0x2, - 0x2, 0x2, 0x375, 0x37b, 0x5, 0xd8, 0x6d, 0x2, 0x376, 0x378, 0x7, 0xcf, + 0x2, 0x2, 0x375, 0x37b, 0x5, 0xd8, 0x6d, 0x2, 0x376, 0x378, 0x7, 0xd0, 0x2, 0x2, 0x377, 0x379, 0x5, 0xac, 0x57, 0x2, 0x378, 0x377, 0x3, 0x2, 0x2, 0x2, 0x378, 0x379, 0x3, 0x2, 0x2, 0x2, 0x379, 0x37a, 0x3, 0x2, - 0x2, 0x2, 0x37a, 0x37c, 0x7, 0xd9, 0x2, 0x2, 0x37b, 0x376, 0x3, 0x2, + 0x2, 0x2, 0x37a, 0x37c, 0x7, 0xda, 0x2, 0x2, 0x37b, 0x376, 0x3, 0x2, 0x2, 0x2, 0x37b, 0x37c, 0x3, 0x2, 0x2, 0x2, 0x37c, 0x41, 0x3, 0x2, 0x2, - 0x2, 0x37d, 0x388, 0x5, 0x44, 0x23, 0x2, 0x37e, 0x37f, 0x7, 0x1d, 0x2, - 0x2, 0x37f, 0x380, 0x5, 0xd6, 0x6c, 0x2, 0x380, 0x381, 0x7, 0x16, 0x2, + 0x2, 0x37d, 0x388, 0x5, 0x44, 0x23, 0x2, 0x37e, 0x37f, 0x7, 0x1e, 0x2, + 0x2, 0x37f, 0x380, 0x5, 0xd6, 0x6c, 0x2, 0x380, 0x381, 0x7, 0x17, 0x2, 0x2, 0x381, 0x382, 0x5, 0xb0, 0x59, 0x2, 0x382, 0x388, 0x3, 0x2, 0x2, - 0x2, 0x383, 0x384, 0x7, 0x4f, 0x2, 0x2, 0x384, 0x388, 0x5, 0x48, 0x25, - 0x2, 0x385, 0x386, 0x7, 0x7f, 0x2, 0x2, 0x386, 0x388, 0x5, 0x4a, 0x26, + 0x2, 0x383, 0x384, 0x7, 0x50, 0x2, 0x2, 0x384, 0x388, 0x5, 0x48, 0x25, + 0x2, 0x385, 0x386, 0x7, 0x80, 0x2, 0x2, 0x386, 0x388, 0x5, 0x4a, 0x26, 0x2, 0x387, 0x37d, 0x3, 0x2, 0x2, 0x2, 0x387, 0x37e, 0x3, 0x2, 0x2, 0x2, 0x387, 0x383, 0x3, 0x2, 0x2, 0x2, 0x387, 0x385, 0x3, 0x2, 0x2, 0x2, 0x388, 0x43, 0x3, 0x2, 0x2, 0x2, 0x389, 0x38a, 0x5, 0xba, 0x5e, 0x2, 0x38a, 0x38c, 0x5, 0xaa, 0x56, 0x2, 0x38b, 0x38d, 0x5, 0x46, 0x24, 0x2, 0x38c, 0x38b, 0x3, 0x2, 0x2, 0x2, 0x38c, 0x38d, 0x3, 0x2, 0x2, - 0x2, 0x38d, 0x390, 0x3, 0x2, 0x2, 0x2, 0x38e, 0x38f, 0x7, 0x1c, 0x2, - 0x2, 0x38f, 0x391, 0x7, 0xbe, 0x2, 0x2, 0x390, 0x38e, 0x3, 0x2, 0x2, + 0x2, 0x38d, 0x390, 0x3, 0x2, 0x2, 0x2, 0x38e, 0x38f, 0x7, 0x1d, 0x2, + 0x2, 0x38f, 0x391, 0x7, 0xbf, 0x2, 0x2, 0x390, 0x38e, 0x3, 0x2, 0x2, 0x2, 0x390, 0x391, 0x3, 0x2, 0x2, 0x2, 0x391, 0x393, 0x3, 0x2, 0x2, 0x2, 0x392, 0x394, 0x5, 0x4c, 0x27, 0x2, 0x393, 0x392, 0x3, 0x2, 0x2, 0x2, 0x393, 0x394, 0x3, 0x2, 0x2, 0x2, 0x394, 0x397, 0x3, 0x2, 0x2, - 0x2, 0x395, 0x396, 0x7, 0xa7, 0x2, 0x2, 0x396, 0x398, 0x5, 0xb0, 0x59, + 0x2, 0x395, 0x396, 0x7, 0xa8, 0x2, 0x2, 0x396, 0x398, 0x5, 0xb0, 0x59, 0x2, 0x397, 0x395, 0x3, 0x2, 0x2, 0x2, 0x397, 0x398, 0x3, 0x2, 0x2, 0x2, 0x398, 0x3aa, 0x3, 0x2, 0x2, 0x2, 0x399, 0x39b, 0x5, 0xba, 0x5e, 0x2, 0x39a, 0x39c, 0x5, 0xaa, 0x56, 0x2, 0x39b, 0x39a, 0x3, 0x2, 0x2, 0x2, 0x39b, 0x39c, 0x3, 0x2, 0x2, 0x2, 0x39c, 0x39d, 0x3, 0x2, 0x2, - 0x2, 0x39d, 0x3a0, 0x5, 0x46, 0x24, 0x2, 0x39e, 0x39f, 0x7, 0x1c, 0x2, - 0x2, 0x39f, 0x3a1, 0x7, 0xbe, 0x2, 0x2, 0x3a0, 0x39e, 0x3, 0x2, 0x2, + 0x2, 0x39d, 0x3a0, 0x5, 0x46, 0x24, 0x2, 0x39e, 0x39f, 0x7, 0x1d, 0x2, + 0x2, 0x39f, 0x3a1, 0x7, 0xbf, 0x2, 0x2, 0x3a0, 0x39e, 0x3, 0x2, 0x2, 0x2, 0x3a0, 0x3a1, 0x3, 0x2, 0x2, 0x2, 0x3a1, 0x3a3, 0x3, 0x2, 0x2, 0x2, 0x3a2, 0x3a4, 0x5, 0x4c, 0x27, 0x2, 0x3a3, 0x3a2, 0x3, 0x2, 0x2, 0x2, 0x3a3, 0x3a4, 0x3, 0x2, 0x2, 0x2, 0x3a4, 0x3a7, 0x3, 0x2, 0x2, - 0x2, 0x3a5, 0x3a6, 0x7, 0xa7, 0x2, 0x2, 0x3a6, 0x3a8, 0x5, 0xb0, 0x59, + 0x2, 0x3a5, 0x3a6, 0x7, 0xa8, 0x2, 0x2, 0x3a6, 0x3a8, 0x5, 0xb0, 0x59, 0x2, 0x3a7, 0x3a5, 0x3, 0x2, 0x2, 0x2, 0x3a7, 0x3a8, 0x3, 0x2, 0x2, 0x2, 0x3a8, 0x3aa, 0x3, 0x2, 0x2, 0x2, 0x3a9, 0x389, 0x3, 0x2, 0x2, 0x2, 0x3a9, 0x399, 0x3, 0x2, 0x2, 0x2, 0x3aa, 0x45, 0x3, 0x2, 0x2, 0x2, 0x3ab, 0x3ac, 0x9, 0x4, 0x2, 0x2, 0x3ac, 0x3ad, 0x5, 0xb0, 0x59, 0x2, 0x3ad, 0x47, 0x3, 0x2, 0x2, 0x2, 0x3ae, 0x3af, 0x5, 0xba, 0x5e, 0x2, - 0x3af, 0x3b0, 0x5, 0xb0, 0x59, 0x2, 0x3b0, 0x3b1, 0x7, 0xa8, 0x2, 0x2, - 0x3b1, 0x3b2, 0x5, 0xaa, 0x56, 0x2, 0x3b2, 0x3b3, 0x7, 0x46, 0x2, 0x2, - 0x3b3, 0x3b4, 0x7, 0xbc, 0x2, 0x2, 0x3b4, 0x49, 0x3, 0x2, 0x2, 0x2, + 0x3af, 0x3b0, 0x5, 0xb0, 0x59, 0x2, 0x3b0, 0x3b1, 0x7, 0xa9, 0x2, 0x2, + 0x3b1, 0x3b2, 0x5, 0xaa, 0x56, 0x2, 0x3b2, 0x3b3, 0x7, 0x47, 0x2, 0x2, + 0x3b3, 0x3b4, 0x7, 0xbd, 0x2, 0x2, 0x3b4, 0x49, 0x3, 0x2, 0x2, 0x2, 0x3b5, 0x3b6, 0x5, 0xba, 0x5e, 0x2, 0x3b6, 0x3b7, 0x5, 0x66, 0x34, 0x2, - 0x3b7, 0x4b, 0x3, 0x2, 0x2, 0x2, 0x3b8, 0x3b9, 0x7, 0x19, 0x2, 0x2, - 0x3b9, 0x3ba, 0x7, 0xcf, 0x2, 0x2, 0x3ba, 0x3bf, 0x5, 0x4e, 0x28, 0x2, - 0x3bb, 0x3bc, 0x7, 0xc4, 0x2, 0x2, 0x3bc, 0x3be, 0x5, 0x4e, 0x28, 0x2, + 0x3b7, 0x4b, 0x3, 0x2, 0x2, 0x2, 0x3b8, 0x3b9, 0x7, 0x1a, 0x2, 0x2, + 0x3b9, 0x3ba, 0x7, 0xd0, 0x2, 0x2, 0x3ba, 0x3bf, 0x5, 0x4e, 0x28, 0x2, + 0x3bb, 0x3bc, 0x7, 0xc5, 0x2, 0x2, 0x3bc, 0x3be, 0x5, 0x4e, 0x28, 0x2, 0x3bd, 0x3bb, 0x3, 0x2, 0x2, 0x2, 0x3be, 0x3c1, 0x3, 0x2, 0x2, 0x2, 0x3bf, 0x3bd, 0x3, 0x2, 0x2, 0x2, 0x3bf, 0x3c0, 0x3, 0x2, 0x2, 0x2, 0x3c0, 0x3c2, 0x3, 0x2, 0x2, 0x2, 0x3c1, 0x3bf, 0x3, 0x2, 0x2, 0x2, - 0x3c2, 0x3c3, 0x7, 0xd9, 0x2, 0x2, 0x3c3, 0x4d, 0x3, 0x2, 0x2, 0x2, - 0x3c4, 0x3ca, 0x5, 0xd6, 0x6c, 0x2, 0x3c5, 0x3c7, 0x7, 0xcf, 0x2, 0x2, + 0x3c2, 0x3c3, 0x7, 0xda, 0x2, 0x2, 0x3c3, 0x4d, 0x3, 0x2, 0x2, 0x2, + 0x3c4, 0x3ca, 0x5, 0xd6, 0x6c, 0x2, 0x3c5, 0x3c7, 0x7, 0xd0, 0x2, 0x2, 0x3c6, 0x3c8, 0x5, 0xac, 0x57, 0x2, 0x3c7, 0x3c6, 0x3, 0x2, 0x2, 0x2, 0x3c7, 0x3c8, 0x3, 0x2, 0x2, 0x2, 0x3c8, 0x3c9, 0x3, 0x2, 0x2, 0x2, - 0x3c9, 0x3cb, 0x7, 0xd9, 0x2, 0x2, 0x3ca, 0x3c5, 0x3, 0x2, 0x2, 0x2, + 0x3c9, 0x3cb, 0x7, 0xda, 0x2, 0x2, 0x3ca, 0x3c5, 0x3, 0x2, 0x2, 0x2, 0x3ca, 0x3cb, 0x3, 0x2, 0x2, 0x2, 0x3cb, 0x4f, 0x3, 0x2, 0x2, 0x2, 0x3cc, - 0x3d4, 0x5, 0xb0, 0x59, 0x2, 0x3cd, 0x3d5, 0x7, 0x28, 0x2, 0x2, 0x3ce, - 0x3cf, 0x7, 0xa1, 0x2, 0x2, 0x3cf, 0x3d0, 0x7, 0x2f, 0x2, 0x2, 0x3d0, - 0x3d5, 0x7, 0xbe, 0x2, 0x2, 0x3d1, 0x3d2, 0x7, 0xa1, 0x2, 0x2, 0x3d2, - 0x3d3, 0x7, 0xb0, 0x2, 0x2, 0x3d3, 0x3d5, 0x7, 0xbe, 0x2, 0x2, 0x3d4, + 0x3d4, 0x5, 0xb0, 0x59, 0x2, 0x3cd, 0x3d5, 0x7, 0x29, 0x2, 0x2, 0x3ce, + 0x3cf, 0x7, 0xa2, 0x2, 0x2, 0x3cf, 0x3d0, 0x7, 0x30, 0x2, 0x2, 0x3d0, + 0x3d5, 0x7, 0xbf, 0x2, 0x2, 0x3d1, 0x3d2, 0x7, 0xa2, 0x2, 0x2, 0x3d2, + 0x3d3, 0x7, 0xb1, 0x2, 0x2, 0x3d3, 0x3d5, 0x7, 0xbf, 0x2, 0x2, 0x3d4, 0x3cd, 0x3, 0x2, 0x2, 0x2, 0x3d4, 0x3ce, 0x3, 0x2, 0x2, 0x2, 0x3d4, 0x3d1, 0x3, 0x2, 0x2, 0x2, 0x3d4, 0x3d5, 0x3, 0x2, 0x2, 0x2, 0x3d5, 0x51, 0x3, 0x2, 0x2, 0x2, 0x3d6, 0x3d8, 0x9, 0x5, 0x2, 0x2, 0x3d7, 0x3d9, - 0x7, 0x99, 0x2, 0x2, 0x3d8, 0x3d7, 0x3, 0x2, 0x2, 0x2, 0x3d8, 0x3d9, + 0x7, 0x9a, 0x2, 0x2, 0x3d8, 0x3d7, 0x3, 0x2, 0x2, 0x2, 0x3d8, 0x3d9, 0x3, 0x2, 0x2, 0x2, 0x3d9, 0x3da, 0x3, 0x2, 0x2, 0x2, 0x3da, 0x3db, 0x5, 0xbc, 0x5f, 0x2, 0x3db, 0x53, 0x3, 0x2, 0x2, 0x2, 0x3dc, 0x3dd, - 0x9, 0x6, 0x2, 0x2, 0x3dd, 0x3e0, 0x7, 0x21, 0x2, 0x2, 0x3de, 0x3df, - 0x7, 0x4c, 0x2, 0x2, 0x3df, 0x3e1, 0x7, 0x37, 0x2, 0x2, 0x3e0, 0x3de, + 0x9, 0x6, 0x2, 0x2, 0x3dd, 0x3e0, 0x7, 0x22, 0x2, 0x2, 0x3de, 0x3df, + 0x7, 0x4d, 0x2, 0x2, 0x3df, 0x3e1, 0x7, 0x38, 0x2, 0x2, 0x3e0, 0x3de, 0x3, 0x2, 0x2, 0x2, 0x3e0, 0x3e1, 0x3, 0x2, 0x2, 0x2, 0x3e1, 0x3e2, 0x3, 0x2, 0x2, 0x2, 0x3e2, 0x3e4, 0x5, 0xc6, 0x64, 0x2, 0x3e3, 0x3e5, 0x5, 0x2c, 0x17, 0x2, 0x3e4, 0x3e3, 0x3, 0x2, 0x2, 0x2, 0x3e4, 0x3e5, 0x3, 0x2, 0x2, 0x2, 0x3e5, 0x3fc, 0x3, 0x2, 0x2, 0x2, 0x3e6, 0x3ed, - 0x9, 0x6, 0x2, 0x2, 0x3e7, 0x3ee, 0x7, 0x2e, 0x2, 0x2, 0x3e8, 0x3ea, - 0x7, 0x9b, 0x2, 0x2, 0x3e9, 0x3e8, 0x3, 0x2, 0x2, 0x2, 0x3e9, 0x3ea, + 0x9, 0x6, 0x2, 0x2, 0x3e7, 0x3ee, 0x7, 0x2f, 0x2, 0x2, 0x3e8, 0x3ea, + 0x7, 0x9c, 0x2, 0x2, 0x3e9, 0x3e8, 0x3, 0x2, 0x2, 0x2, 0x3e9, 0x3ea, 0x3, 0x2, 0x2, 0x2, 0x3ea, 0x3eb, 0x3, 0x2, 0x2, 0x2, 0x3eb, 0x3ee, - 0x7, 0x99, 0x2, 0x2, 0x3ec, 0x3ee, 0x7, 0xaf, 0x2, 0x2, 0x3ed, 0x3e7, + 0x7, 0x9a, 0x2, 0x2, 0x3ec, 0x3ee, 0x7, 0xb0, 0x2, 0x2, 0x3ed, 0x3e7, 0x3, 0x2, 0x2, 0x2, 0x3ed, 0x3e9, 0x3, 0x2, 0x2, 0x2, 0x3ed, 0x3ec, 0x3, 0x2, 0x2, 0x2, 0x3ee, 0x3f1, 0x3, 0x2, 0x2, 0x2, 0x3ef, 0x3f0, - 0x7, 0x4c, 0x2, 0x2, 0x3f0, 0x3f2, 0x7, 0x37, 0x2, 0x2, 0x3f1, 0x3ef, + 0x7, 0x4d, 0x2, 0x2, 0x3f0, 0x3f2, 0x7, 0x38, 0x2, 0x2, 0x3f1, 0x3ef, 0x3, 0x2, 0x2, 0x2, 0x3f1, 0x3f2, 0x3, 0x2, 0x2, 0x2, 0x3f2, 0x3f3, 0x3, 0x2, 0x2, 0x2, 0x3f3, 0x3f5, 0x5, 0xc0, 0x61, 0x2, 0x3f4, 0x3f6, 0x5, 0x2c, 0x17, 0x2, 0x3f5, 0x3f4, 0x3, 0x2, 0x2, 0x2, 0x3f5, 0x3f6, 0x3, 0x2, 0x2, 0x2, 0x3f6, 0x3f9, 0x3, 0x2, 0x2, 0x2, 0x3f7, 0x3f8, - 0x7, 0x70, 0x2, 0x2, 0x3f8, 0x3fa, 0x7, 0x27, 0x2, 0x2, 0x3f9, 0x3f7, + 0x7, 0x71, 0x2, 0x2, 0x3f8, 0x3fa, 0x7, 0x28, 0x2, 0x2, 0x3f9, 0x3f7, 0x3, 0x2, 0x2, 0x2, 0x3f9, 0x3fa, 0x3, 0x2, 0x2, 0x2, 0x3fa, 0x3fc, 0x3, 0x2, 0x2, 0x2, 0x3fb, 0x3dc, 0x3, 0x2, 0x2, 0x2, 0x3fb, 0x3e6, 0x3, 0x2, 0x2, 0x2, 0x3fc, 0x55, 0x3, 0x2, 0x2, 0x2, 0x3fd, 0x3fe, 0x7, - 0x37, 0x2, 0x2, 0x3fe, 0x3ff, 0x7, 0x21, 0x2, 0x2, 0x3ff, 0x40b, 0x5, - 0xc6, 0x64, 0x2, 0x400, 0x407, 0x7, 0x37, 0x2, 0x2, 0x401, 0x408, 0x7, - 0x2e, 0x2, 0x2, 0x402, 0x404, 0x7, 0x9b, 0x2, 0x2, 0x403, 0x402, 0x3, + 0x38, 0x2, 0x2, 0x3fe, 0x3ff, 0x7, 0x22, 0x2, 0x2, 0x3ff, 0x40b, 0x5, + 0xc6, 0x64, 0x2, 0x400, 0x407, 0x7, 0x38, 0x2, 0x2, 0x401, 0x408, 0x7, + 0x2f, 0x2, 0x2, 0x402, 0x404, 0x7, 0x9c, 0x2, 0x2, 0x403, 0x402, 0x3, 0x2, 0x2, 0x2, 0x403, 0x404, 0x3, 0x2, 0x2, 0x2, 0x404, 0x405, 0x3, - 0x2, 0x2, 0x2, 0x405, 0x408, 0x7, 0x99, 0x2, 0x2, 0x406, 0x408, 0x7, - 0xaf, 0x2, 0x2, 0x407, 0x401, 0x3, 0x2, 0x2, 0x2, 0x407, 0x403, 0x3, + 0x2, 0x2, 0x2, 0x405, 0x408, 0x7, 0x9a, 0x2, 0x2, 0x406, 0x408, 0x7, + 0xb0, 0x2, 0x2, 0x407, 0x401, 0x3, 0x2, 0x2, 0x2, 0x407, 0x403, 0x3, 0x2, 0x2, 0x2, 0x407, 0x406, 0x3, 0x2, 0x2, 0x2, 0x407, 0x408, 0x3, 0x2, 0x2, 0x2, 0x408, 0x409, 0x3, 0x2, 0x2, 0x2, 0x409, 0x40b, 0x5, 0xc0, 0x61, 0x2, 0x40a, 0x3fd, 0x3, 0x2, 0x2, 0x2, 0x40a, 0x400, 0x3, - 0x2, 0x2, 0x2, 0x40b, 0x57, 0x3, 0x2, 0x2, 0x2, 0x40c, 0x40d, 0x7, 0x38, - 0x2, 0x2, 0x40d, 0x40e, 0x7, 0x97, 0x2, 0x2, 0x40e, 0x40f, 0x5, 0x4, - 0x3, 0x2, 0x40f, 0x59, 0x3, 0x2, 0x2, 0x2, 0x410, 0x411, 0x7, 0x53, - 0x2, 0x2, 0x411, 0x413, 0x7, 0x55, 0x2, 0x2, 0x412, 0x414, 0x7, 0x99, - 0x2, 0x2, 0x413, 0x412, 0x3, 0x2, 0x2, 0x2, 0x413, 0x414, 0x3, 0x2, - 0x2, 0x2, 0x414, 0x418, 0x3, 0x2, 0x2, 0x2, 0x415, 0x419, 0x5, 0xc0, - 0x61, 0x2, 0x416, 0x417, 0x7, 0x44, 0x2, 0x2, 0x417, 0x419, 0x5, 0xbe, - 0x60, 0x2, 0x418, 0x415, 0x3, 0x2, 0x2, 0x2, 0x418, 0x416, 0x3, 0x2, - 0x2, 0x2, 0x419, 0x41b, 0x3, 0x2, 0x2, 0x2, 0x41a, 0x41c, 0x5, 0x5c, - 0x2f, 0x2, 0x41b, 0x41a, 0x3, 0x2, 0x2, 0x2, 0x41b, 0x41c, 0x3, 0x2, - 0x2, 0x2, 0x41c, 0x41d, 0x3, 0x2, 0x2, 0x2, 0x41d, 0x41e, 0x5, 0x5e, - 0x30, 0x2, 0x41e, 0x5b, 0x3, 0x2, 0x2, 0x2, 0x41f, 0x420, 0x7, 0xcf, - 0x2, 0x2, 0x420, 0x425, 0x5, 0xba, 0x5e, 0x2, 0x421, 0x422, 0x7, 0xc4, - 0x2, 0x2, 0x422, 0x424, 0x5, 0xba, 0x5e, 0x2, 0x423, 0x421, 0x3, 0x2, - 0x2, 0x2, 0x424, 0x427, 0x3, 0x2, 0x2, 0x2, 0x425, 0x423, 0x3, 0x2, - 0x2, 0x2, 0x425, 0x426, 0x3, 0x2, 0x2, 0x2, 0x426, 0x428, 0x3, 0x2, - 0x2, 0x2, 0x427, 0x425, 0x3, 0x2, 0x2, 0x2, 0x428, 0x429, 0x7, 0xd9, - 0x2, 0x2, 0x429, 0x5d, 0x3, 0x2, 0x2, 0x2, 0x42a, 0x42b, 0x7, 0x40, - 0x2, 0x2, 0x42b, 0x434, 0x5, 0xd6, 0x6c, 0x2, 0x42c, 0x434, 0x7, 0xae, - 0x2, 0x2, 0x42d, 0x42f, 0x5, 0x68, 0x35, 0x2, 0x42e, 0x430, 0x7, 0xda, - 0x2, 0x2, 0x42f, 0x42e, 0x3, 0x2, 0x2, 0x2, 0x42f, 0x430, 0x3, 0x2, - 0x2, 0x2, 0x430, 0x431, 0x3, 0x2, 0x2, 0x2, 0x431, 0x432, 0x7, 0x2, - 0x2, 0x3, 0x432, 0x434, 0x3, 0x2, 0x2, 0x2, 0x433, 0x42a, 0x3, 0x2, - 0x2, 0x2, 0x433, 0x42c, 0x3, 0x2, 0x2, 0x2, 0x433, 0x42d, 0x3, 0x2, - 0x2, 0x2, 0x434, 0x5f, 0x3, 0x2, 0x2, 0x2, 0x435, 0x436, 0x7, 0x5a, - 0x2, 0x2, 0x436, 0x438, 0x7, 0x6e, 0x2, 0x2, 0x437, 0x439, 0x5, 0x2c, - 0x17, 0x2, 0x438, 0x437, 0x3, 0x2, 0x2, 0x2, 0x438, 0x439, 0x3, 0x2, - 0x2, 0x2, 0x439, 0x43a, 0x3, 0x2, 0x2, 0x2, 0x43a, 0x43c, 0x5, 0x78, - 0x3d, 0x2, 0x43b, 0x43d, 0x9, 0x7, 0x2, 0x2, 0x43c, 0x43b, 0x3, 0x2, - 0x2, 0x2, 0x43c, 0x43d, 0x3, 0x2, 0x2, 0x2, 0x43d, 0x61, 0x3, 0x2, 0x2, - 0x2, 0x43e, 0x43f, 0x7, 0x76, 0x2, 0x2, 0x43f, 0x440, 0x7, 0x99, 0x2, - 0x2, 0x440, 0x442, 0x5, 0xc0, 0x61, 0x2, 0x441, 0x443, 0x5, 0x2c, 0x17, - 0x2, 0x442, 0x441, 0x3, 0x2, 0x2, 0x2, 0x442, 0x443, 0x3, 0x2, 0x2, - 0x2, 0x443, 0x445, 0x3, 0x2, 0x2, 0x2, 0x444, 0x446, 0x5, 0x10, 0x9, - 0x2, 0x445, 0x444, 0x3, 0x2, 0x2, 0x2, 0x445, 0x446, 0x3, 0x2, 0x2, - 0x2, 0x446, 0x448, 0x3, 0x2, 0x2, 0x2, 0x447, 0x449, 0x7, 0x3c, 0x2, - 0x2, 0x448, 0x447, 0x3, 0x2, 0x2, 0x2, 0x448, 0x449, 0x3, 0x2, 0x2, - 0x2, 0x449, 0x44b, 0x3, 0x2, 0x2, 0x2, 0x44a, 0x44c, 0x7, 0x25, 0x2, - 0x2, 0x44b, 0x44a, 0x3, 0x2, 0x2, 0x2, 0x44b, 0x44c, 0x3, 0x2, 0x2, - 0x2, 0x44c, 0x63, 0x3, 0x2, 0x2, 0x2, 0x44d, 0x44e, 0x7, 0x84, 0x2, - 0x2, 0x44e, 0x44f, 0x7, 0x99, 0x2, 0x2, 0x44f, 0x450, 0x5, 0xc0, 0x61, - 0x2, 0x450, 0x451, 0x7, 0xa1, 0x2, 0x2, 0x451, 0x459, 0x5, 0xc0, 0x61, - 0x2, 0x452, 0x453, 0x7, 0xc4, 0x2, 0x2, 0x453, 0x454, 0x5, 0xc0, 0x61, - 0x2, 0x454, 0x455, 0x7, 0xa1, 0x2, 0x2, 0x455, 0x456, 0x5, 0xc0, 0x61, - 0x2, 0x456, 0x458, 0x3, 0x2, 0x2, 0x2, 0x457, 0x452, 0x3, 0x2, 0x2, - 0x2, 0x458, 0x45b, 0x3, 0x2, 0x2, 0x2, 0x459, 0x457, 0x3, 0x2, 0x2, - 0x2, 0x459, 0x45a, 0x3, 0x2, 0x2, 0x2, 0x45a, 0x45d, 0x3, 0x2, 0x2, - 0x2, 0x45b, 0x459, 0x3, 0x2, 0x2, 0x2, 0x45c, 0x45e, 0x5, 0x2c, 0x17, - 0x2, 0x45d, 0x45c, 0x3, 0x2, 0x2, 0x2, 0x45d, 0x45e, 0x3, 0x2, 0x2, - 0x2, 0x45e, 0x65, 0x3, 0x2, 0x2, 0x2, 0x45f, 0x461, 0x7, 0xcf, 0x2, - 0x2, 0x460, 0x462, 0x5, 0x6e, 0x38, 0x2, 0x461, 0x460, 0x3, 0x2, 0x2, - 0x2, 0x461, 0x462, 0x3, 0x2, 0x2, 0x2, 0x462, 0x463, 0x3, 0x2, 0x2, - 0x2, 0x463, 0x464, 0x7, 0x8c, 0x2, 0x2, 0x464, 0x466, 0x5, 0xac, 0x57, - 0x2, 0x465, 0x467, 0x5, 0x7a, 0x3e, 0x2, 0x466, 0x465, 0x3, 0x2, 0x2, - 0x2, 0x466, 0x467, 0x3, 0x2, 0x2, 0x2, 0x467, 0x469, 0x3, 0x2, 0x2, - 0x2, 0x468, 0x46a, 0x5, 0x80, 0x41, 0x2, 0x469, 0x468, 0x3, 0x2, 0x2, - 0x2, 0x469, 0x46a, 0x3, 0x2, 0x2, 0x2, 0x46a, 0x46b, 0x3, 0x2, 0x2, - 0x2, 0x46b, 0x46c, 0x7, 0xd9, 0x2, 0x2, 0x46c, 0x67, 0x3, 0x2, 0x2, - 0x2, 0x46d, 0x473, 0x5, 0x6a, 0x36, 0x2, 0x46e, 0x46f, 0x7, 0xa9, 0x2, - 0x2, 0x46f, 0x470, 0x7, 0x6, 0x2, 0x2, 0x470, 0x472, 0x5, 0x6a, 0x36, - 0x2, 0x471, 0x46e, 0x3, 0x2, 0x2, 0x2, 0x472, 0x475, 0x3, 0x2, 0x2, - 0x2, 0x473, 0x471, 0x3, 0x2, 0x2, 0x2, 0x473, 0x474, 0x3, 0x2, 0x2, - 0x2, 0x474, 0x69, 0x3, 0x2, 0x2, 0x2, 0x475, 0x473, 0x3, 0x2, 0x2, 0x2, - 0x476, 0x47c, 0x5, 0x6c, 0x37, 0x2, 0x477, 0x478, 0x7, 0xcf, 0x2, 0x2, - 0x478, 0x479, 0x5, 0x68, 0x35, 0x2, 0x479, 0x47a, 0x7, 0xd9, 0x2, 0x2, - 0x47a, 0x47c, 0x3, 0x2, 0x2, 0x2, 0x47b, 0x476, 0x3, 0x2, 0x2, 0x2, - 0x47b, 0x477, 0x3, 0x2, 0x2, 0x2, 0x47c, 0x6b, 0x3, 0x2, 0x2, 0x2, 0x47d, - 0x47f, 0x5, 0x6e, 0x38, 0x2, 0x47e, 0x47d, 0x3, 0x2, 0x2, 0x2, 0x47e, - 0x47f, 0x3, 0x2, 0x2, 0x2, 0x47f, 0x480, 0x3, 0x2, 0x2, 0x2, 0x480, - 0x482, 0x7, 0x8c, 0x2, 0x2, 0x481, 0x483, 0x7, 0x30, 0x2, 0x2, 0x482, - 0x481, 0x3, 0x2, 0x2, 0x2, 0x482, 0x483, 0x3, 0x2, 0x2, 0x2, 0x483, - 0x485, 0x3, 0x2, 0x2, 0x2, 0x484, 0x486, 0x5, 0x70, 0x39, 0x2, 0x485, - 0x484, 0x3, 0x2, 0x2, 0x2, 0x485, 0x486, 0x3, 0x2, 0x2, 0x2, 0x486, - 0x487, 0x3, 0x2, 0x2, 0x2, 0x487, 0x489, 0x5, 0xac, 0x57, 0x2, 0x488, - 0x48a, 0x5, 0x72, 0x3a, 0x2, 0x489, 0x488, 0x3, 0x2, 0x2, 0x2, 0x489, - 0x48a, 0x3, 0x2, 0x2, 0x2, 0x48a, 0x48c, 0x3, 0x2, 0x2, 0x2, 0x48b, - 0x48d, 0x5, 0x74, 0x3b, 0x2, 0x48c, 0x48b, 0x3, 0x2, 0x2, 0x2, 0x48c, - 0x48d, 0x3, 0x2, 0x2, 0x2, 0x48d, 0x48f, 0x3, 0x2, 0x2, 0x2, 0x48e, - 0x490, 0x5, 0x76, 0x3c, 0x2, 0x48f, 0x48e, 0x3, 0x2, 0x2, 0x2, 0x48f, - 0x490, 0x3, 0x2, 0x2, 0x2, 0x490, 0x492, 0x3, 0x2, 0x2, 0x2, 0x491, - 0x493, 0x5, 0x78, 0x3d, 0x2, 0x492, 0x491, 0x3, 0x2, 0x2, 0x2, 0x492, - 0x493, 0x3, 0x2, 0x2, 0x2, 0x493, 0x495, 0x3, 0x2, 0x2, 0x2, 0x494, - 0x496, 0x5, 0x7a, 0x3e, 0x2, 0x495, 0x494, 0x3, 0x2, 0x2, 0x2, 0x495, - 0x496, 0x3, 0x2, 0x2, 0x2, 0x496, 0x499, 0x3, 0x2, 0x2, 0x2, 0x497, - 0x498, 0x7, 0xb5, 0x2, 0x2, 0x498, 0x49a, 0x9, 0x8, 0x2, 0x2, 0x499, - 0x497, 0x3, 0x2, 0x2, 0x2, 0x499, 0x49a, 0x3, 0x2, 0x2, 0x2, 0x49a, - 0x49d, 0x3, 0x2, 0x2, 0x2, 0x49b, 0x49c, 0x7, 0xb5, 0x2, 0x2, 0x49c, - 0x49e, 0x7, 0xa3, 0x2, 0x2, 0x49d, 0x49b, 0x3, 0x2, 0x2, 0x2, 0x49d, - 0x49e, 0x3, 0x2, 0x2, 0x2, 0x49e, 0x4a0, 0x3, 0x2, 0x2, 0x2, 0x49f, - 0x4a1, 0x5, 0x7c, 0x3f, 0x2, 0x4a0, 0x49f, 0x3, 0x2, 0x2, 0x2, 0x4a0, - 0x4a1, 0x3, 0x2, 0x2, 0x2, 0x4a1, 0x4a3, 0x3, 0x2, 0x2, 0x2, 0x4a2, - 0x4a4, 0x5, 0x7e, 0x40, 0x2, 0x4a3, 0x4a2, 0x3, 0x2, 0x2, 0x2, 0x4a3, - 0x4a4, 0x3, 0x2, 0x2, 0x2, 0x4a4, 0x4a6, 0x3, 0x2, 0x2, 0x2, 0x4a5, - 0x4a7, 0x5, 0x82, 0x42, 0x2, 0x4a6, 0x4a5, 0x3, 0x2, 0x2, 0x2, 0x4a6, - 0x4a7, 0x3, 0x2, 0x2, 0x2, 0x4a7, 0x4a9, 0x3, 0x2, 0x2, 0x2, 0x4a8, - 0x4aa, 0x5, 0x84, 0x43, 0x2, 0x4a9, 0x4a8, 0x3, 0x2, 0x2, 0x2, 0x4a9, - 0x4aa, 0x3, 0x2, 0x2, 0x2, 0x4aa, 0x4ac, 0x3, 0x2, 0x2, 0x2, 0x4ab, - 0x4ad, 0x5, 0x86, 0x44, 0x2, 0x4ac, 0x4ab, 0x3, 0x2, 0x2, 0x2, 0x4ac, - 0x4ad, 0x3, 0x2, 0x2, 0x2, 0x4ad, 0x6d, 0x3, 0x2, 0x2, 0x2, 0x4ae, 0x4af, - 0x7, 0xb5, 0x2, 0x2, 0x4af, 0x4b0, 0x5, 0xac, 0x57, 0x2, 0x4b0, 0x6f, - 0x3, 0x2, 0x2, 0x2, 0x4b1, 0x4b2, 0x7, 0xa2, 0x2, 0x2, 0x4b2, 0x4b5, - 0x7, 0xbc, 0x2, 0x2, 0x4b3, 0x4b4, 0x7, 0xb5, 0x2, 0x2, 0x4b4, 0x4b6, - 0x7, 0x9e, 0x2, 0x2, 0x4b5, 0x4b3, 0x3, 0x2, 0x2, 0x2, 0x4b5, 0x4b6, - 0x3, 0x2, 0x2, 0x2, 0x4b6, 0x71, 0x3, 0x2, 0x2, 0x2, 0x4b7, 0x4b8, 0x7, - 0x42, 0x2, 0x2, 0x4b8, 0x4b9, 0x5, 0x88, 0x45, 0x2, 0x4b9, 0x73, 0x3, - 0x2, 0x2, 0x2, 0x4ba, 0x4bc, 0x9, 0x9, 0x2, 0x2, 0x4bb, 0x4ba, 0x3, - 0x2, 0x2, 0x2, 0x4bb, 0x4bc, 0x3, 0x2, 0x2, 0x2, 0x4bc, 0x4bd, 0x3, - 0x2, 0x2, 0x2, 0x4bd, 0x4be, 0x7, 0xb, 0x2, 0x2, 0x4be, 0x4bf, 0x7, - 0x58, 0x2, 0x2, 0x4bf, 0x4c0, 0x5, 0xac, 0x57, 0x2, 0x4c0, 0x75, 0x3, - 0x2, 0x2, 0x2, 0x4c1, 0x4c2, 0x7, 0x7d, 0x2, 0x2, 0x4c2, 0x4c3, 0x5, - 0xb0, 0x59, 0x2, 0x4c3, 0x77, 0x3, 0x2, 0x2, 0x2, 0x4c4, 0x4c5, 0x7, - 0xb4, 0x2, 0x2, 0x4c5, 0x4c6, 0x5, 0xb0, 0x59, 0x2, 0x4c6, 0x79, 0x3, - 0x2, 0x2, 0x2, 0x4c7, 0x4c8, 0x7, 0x47, 0x2, 0x2, 0x4c8, 0x4cf, 0x7, - 0x13, 0x2, 0x2, 0x4c9, 0x4ca, 0x9, 0x8, 0x2, 0x2, 0x4ca, 0x4cb, 0x7, - 0xcf, 0x2, 0x2, 0x4cb, 0x4cc, 0x5, 0xac, 0x57, 0x2, 0x4cc, 0x4cd, 0x7, - 0xd9, 0x2, 0x2, 0x4cd, 0x4d0, 0x3, 0x2, 0x2, 0x2, 0x4ce, 0x4d0, 0x5, - 0xac, 0x57, 0x2, 0x4cf, 0x4c9, 0x3, 0x2, 0x2, 0x2, 0x4cf, 0x4ce, 0x3, - 0x2, 0x2, 0x2, 0x4d0, 0x7b, 0x3, 0x2, 0x2, 0x2, 0x4d1, 0x4d2, 0x7, 0x48, - 0x2, 0x2, 0x4d2, 0x4d3, 0x5, 0xb0, 0x59, 0x2, 0x4d3, 0x7d, 0x3, 0x2, - 0x2, 0x2, 0x4d4, 0x4d5, 0x7, 0x78, 0x2, 0x2, 0x4d5, 0x4d6, 0x7, 0x13, - 0x2, 0x2, 0x4d6, 0x4d7, 0x5, 0x94, 0x4b, 0x2, 0x4d7, 0x7f, 0x3, 0x2, - 0x2, 0x2, 0x4d8, 0x4d9, 0x7, 0x78, 0x2, 0x2, 0x4d9, 0x4da, 0x7, 0x13, - 0x2, 0x2, 0x4da, 0x4db, 0x5, 0xac, 0x57, 0x2, 0x4db, 0x81, 0x3, 0x2, - 0x2, 0x2, 0x4dc, 0x4dd, 0x7, 0x61, 0x2, 0x2, 0x4dd, 0x4de, 0x5, 0x92, - 0x4a, 0x2, 0x4de, 0x4df, 0x7, 0x13, 0x2, 0x2, 0x4df, 0x4e0, 0x5, 0xac, - 0x57, 0x2, 0x4e0, 0x83, 0x3, 0x2, 0x2, 0x2, 0x4e1, 0x4e2, 0x7, 0x61, - 0x2, 0x2, 0x4e2, 0x4e5, 0x5, 0x92, 0x4a, 0x2, 0x4e3, 0x4e4, 0x7, 0xb5, - 0x2, 0x2, 0x4e4, 0x4e6, 0x7, 0x9e, 0x2, 0x2, 0x4e5, 0x4e3, 0x3, 0x2, - 0x2, 0x2, 0x4e5, 0x4e6, 0x3, 0x2, 0x2, 0x2, 0x4e6, 0x85, 0x3, 0x2, 0x2, - 0x2, 0x4e7, 0x4e8, 0x7, 0x90, 0x2, 0x2, 0x4e8, 0x4e9, 0x5, 0x9a, 0x4e, - 0x2, 0x4e9, 0x87, 0x3, 0x2, 0x2, 0x2, 0x4ea, 0x4eb, 0x8, 0x45, 0x1, - 0x2, 0x4eb, 0x4ed, 0x5, 0xbc, 0x5f, 0x2, 0x4ec, 0x4ee, 0x7, 0x3c, 0x2, - 0x2, 0x4ed, 0x4ec, 0x3, 0x2, 0x2, 0x2, 0x4ed, 0x4ee, 0x3, 0x2, 0x2, - 0x2, 0x4ee, 0x4f0, 0x3, 0x2, 0x2, 0x2, 0x4ef, 0x4f1, 0x5, 0x90, 0x49, - 0x2, 0x4f0, 0x4ef, 0x3, 0x2, 0x2, 0x2, 0x4f0, 0x4f1, 0x3, 0x2, 0x2, - 0x2, 0x4f1, 0x4f7, 0x3, 0x2, 0x2, 0x2, 0x4f2, 0x4f3, 0x7, 0xcf, 0x2, - 0x2, 0x4f3, 0x4f4, 0x5, 0x88, 0x45, 0x2, 0x4f4, 0x4f5, 0x7, 0xd9, 0x2, - 0x2, 0x4f5, 0x4f7, 0x3, 0x2, 0x2, 0x2, 0x4f6, 0x4ea, 0x3, 0x2, 0x2, - 0x2, 0x4f6, 0x4f2, 0x3, 0x2, 0x2, 0x2, 0x4f7, 0x509, 0x3, 0x2, 0x2, - 0x2, 0x4f8, 0x4f9, 0xc, 0x5, 0x2, 0x2, 0x4f9, 0x4fa, 0x5, 0x8c, 0x47, - 0x2, 0x4fa, 0x4fb, 0x5, 0x88, 0x45, 0x6, 0x4fb, 0x508, 0x3, 0x2, 0x2, - 0x2, 0x4fc, 0x4fe, 0xc, 0x6, 0x2, 0x2, 0x4fd, 0x4ff, 0x9, 0xa, 0x2, - 0x2, 0x4fe, 0x4fd, 0x3, 0x2, 0x2, 0x2, 0x4fe, 0x4ff, 0x3, 0x2, 0x2, - 0x2, 0x4ff, 0x501, 0x3, 0x2, 0x2, 0x2, 0x500, 0x502, 0x5, 0x8a, 0x46, - 0x2, 0x501, 0x500, 0x3, 0x2, 0x2, 0x2, 0x501, 0x502, 0x3, 0x2, 0x2, - 0x2, 0x502, 0x503, 0x3, 0x2, 0x2, 0x2, 0x503, 0x504, 0x7, 0x58, 0x2, - 0x2, 0x504, 0x505, 0x5, 0x88, 0x45, 0x2, 0x505, 0x506, 0x5, 0x8e, 0x48, - 0x2, 0x506, 0x508, 0x3, 0x2, 0x2, 0x2, 0x507, 0x4f8, 0x3, 0x2, 0x2, - 0x2, 0x507, 0x4fc, 0x3, 0x2, 0x2, 0x2, 0x508, 0x50b, 0x3, 0x2, 0x2, - 0x2, 0x509, 0x507, 0x3, 0x2, 0x2, 0x2, 0x509, 0x50a, 0x3, 0x2, 0x2, - 0x2, 0x50a, 0x89, 0x3, 0x2, 0x2, 0x2, 0x50b, 0x509, 0x3, 0x2, 0x2, 0x2, - 0x50c, 0x50e, 0x9, 0xb, 0x2, 0x2, 0x50d, 0x50c, 0x3, 0x2, 0x2, 0x2, - 0x50d, 0x50e, 0x3, 0x2, 0x2, 0x2, 0x50e, 0x50f, 0x3, 0x2, 0x2, 0x2, - 0x50f, 0x516, 0x7, 0x52, 0x2, 0x2, 0x510, 0x512, 0x7, 0x52, 0x2, 0x2, - 0x511, 0x513, 0x9, 0xb, 0x2, 0x2, 0x512, 0x511, 0x3, 0x2, 0x2, 0x2, - 0x512, 0x513, 0x3, 0x2, 0x2, 0x2, 0x513, 0x516, 0x3, 0x2, 0x2, 0x2, - 0x514, 0x516, 0x9, 0xb, 0x2, 0x2, 0x515, 0x50d, 0x3, 0x2, 0x2, 0x2, - 0x515, 0x510, 0x3, 0x2, 0x2, 0x2, 0x515, 0x514, 0x3, 0x2, 0x2, 0x2, - 0x516, 0x538, 0x3, 0x2, 0x2, 0x2, 0x517, 0x519, 0x9, 0xc, 0x2, 0x2, - 0x518, 0x517, 0x3, 0x2, 0x2, 0x2, 0x518, 0x519, 0x3, 0x2, 0x2, 0x2, - 0x519, 0x51a, 0x3, 0x2, 0x2, 0x2, 0x51a, 0x51c, 0x9, 0xd, 0x2, 0x2, - 0x51b, 0x51d, 0x7, 0x79, 0x2, 0x2, 0x51c, 0x51b, 0x3, 0x2, 0x2, 0x2, - 0x51c, 0x51d, 0x3, 0x2, 0x2, 0x2, 0x51d, 0x526, 0x3, 0x2, 0x2, 0x2, - 0x51e, 0x520, 0x9, 0xd, 0x2, 0x2, 0x51f, 0x521, 0x7, 0x79, 0x2, 0x2, - 0x520, 0x51f, 0x3, 0x2, 0x2, 0x2, 0x520, 0x521, 0x3, 0x2, 0x2, 0x2, - 0x521, 0x523, 0x3, 0x2, 0x2, 0x2, 0x522, 0x524, 0x9, 0xc, 0x2, 0x2, - 0x523, 0x522, 0x3, 0x2, 0x2, 0x2, 0x523, 0x524, 0x3, 0x2, 0x2, 0x2, - 0x524, 0x526, 0x3, 0x2, 0x2, 0x2, 0x525, 0x518, 0x3, 0x2, 0x2, 0x2, - 0x525, 0x51e, 0x3, 0x2, 0x2, 0x2, 0x526, 0x538, 0x3, 0x2, 0x2, 0x2, - 0x527, 0x529, 0x9, 0xe, 0x2, 0x2, 0x528, 0x527, 0x3, 0x2, 0x2, 0x2, - 0x528, 0x529, 0x3, 0x2, 0x2, 0x2, 0x529, 0x52a, 0x3, 0x2, 0x2, 0x2, - 0x52a, 0x52c, 0x7, 0x43, 0x2, 0x2, 0x52b, 0x52d, 0x7, 0x79, 0x2, 0x2, - 0x52c, 0x52b, 0x3, 0x2, 0x2, 0x2, 0x52c, 0x52d, 0x3, 0x2, 0x2, 0x2, - 0x52d, 0x536, 0x3, 0x2, 0x2, 0x2, 0x52e, 0x530, 0x7, 0x43, 0x2, 0x2, - 0x52f, 0x531, 0x7, 0x79, 0x2, 0x2, 0x530, 0x52f, 0x3, 0x2, 0x2, 0x2, - 0x530, 0x531, 0x3, 0x2, 0x2, 0x2, 0x531, 0x533, 0x3, 0x2, 0x2, 0x2, - 0x532, 0x534, 0x9, 0xe, 0x2, 0x2, 0x533, 0x532, 0x3, 0x2, 0x2, 0x2, - 0x533, 0x534, 0x3, 0x2, 0x2, 0x2, 0x534, 0x536, 0x3, 0x2, 0x2, 0x2, - 0x535, 0x528, 0x3, 0x2, 0x2, 0x2, 0x535, 0x52e, 0x3, 0x2, 0x2, 0x2, - 0x536, 0x538, 0x3, 0x2, 0x2, 0x2, 0x537, 0x515, 0x3, 0x2, 0x2, 0x2, - 0x537, 0x525, 0x3, 0x2, 0x2, 0x2, 0x537, 0x535, 0x3, 0x2, 0x2, 0x2, - 0x538, 0x8b, 0x3, 0x2, 0x2, 0x2, 0x539, 0x53b, 0x9, 0xa, 0x2, 0x2, 0x53a, - 0x539, 0x3, 0x2, 0x2, 0x2, 0x53a, 0x53b, 0x3, 0x2, 0x2, 0x2, 0x53b, - 0x53c, 0x3, 0x2, 0x2, 0x2, 0x53c, 0x53d, 0x7, 0x1f, 0x2, 0x2, 0x53d, - 0x540, 0x7, 0x58, 0x2, 0x2, 0x53e, 0x540, 0x7, 0xc4, 0x2, 0x2, 0x53f, - 0x53a, 0x3, 0x2, 0x2, 0x2, 0x53f, 0x53e, 0x3, 0x2, 0x2, 0x2, 0x540, - 0x8d, 0x3, 0x2, 0x2, 0x2, 0x541, 0x542, 0x7, 0x75, 0x2, 0x2, 0x542, - 0x54b, 0x5, 0xac, 0x57, 0x2, 0x543, 0x544, 0x7, 0xac, 0x2, 0x2, 0x544, - 0x545, 0x7, 0xcf, 0x2, 0x2, 0x545, 0x546, 0x5, 0xac, 0x57, 0x2, 0x546, - 0x547, 0x7, 0xd9, 0x2, 0x2, 0x547, 0x54b, 0x3, 0x2, 0x2, 0x2, 0x548, - 0x549, 0x7, 0xac, 0x2, 0x2, 0x549, 0x54b, 0x5, 0xac, 0x57, 0x2, 0x54a, - 0x541, 0x3, 0x2, 0x2, 0x2, 0x54a, 0x543, 0x3, 0x2, 0x2, 0x2, 0x54a, - 0x548, 0x3, 0x2, 0x2, 0x2, 0x54b, 0x8f, 0x3, 0x2, 0x2, 0x2, 0x54c, 0x54d, - 0x7, 0x8a, 0x2, 0x2, 0x54d, 0x550, 0x5, 0x98, 0x4d, 0x2, 0x54e, 0x54f, - 0x7, 0x74, 0x2, 0x2, 0x54f, 0x551, 0x5, 0x98, 0x4d, 0x2, 0x550, 0x54e, - 0x3, 0x2, 0x2, 0x2, 0x550, 0x551, 0x3, 0x2, 0x2, 0x2, 0x551, 0x91, 0x3, - 0x2, 0x2, 0x2, 0x552, 0x555, 0x5, 0xb0, 0x59, 0x2, 0x553, 0x554, 0x9, - 0xf, 0x2, 0x2, 0x554, 0x556, 0x5, 0xb0, 0x59, 0x2, 0x555, 0x553, 0x3, - 0x2, 0x2, 0x2, 0x555, 0x556, 0x3, 0x2, 0x2, 0x2, 0x556, 0x93, 0x3, 0x2, - 0x2, 0x2, 0x557, 0x55c, 0x5, 0x96, 0x4c, 0x2, 0x558, 0x559, 0x7, 0xc4, - 0x2, 0x2, 0x559, 0x55b, 0x5, 0x96, 0x4c, 0x2, 0x55a, 0x558, 0x3, 0x2, - 0x2, 0x2, 0x55b, 0x55e, 0x3, 0x2, 0x2, 0x2, 0x55c, 0x55a, 0x3, 0x2, - 0x2, 0x2, 0x55c, 0x55d, 0x3, 0x2, 0x2, 0x2, 0x55d, 0x95, 0x3, 0x2, 0x2, - 0x2, 0x55e, 0x55c, 0x3, 0x2, 0x2, 0x2, 0x55f, 0x561, 0x5, 0xb0, 0x59, - 0x2, 0x560, 0x562, 0x9, 0x10, 0x2, 0x2, 0x561, 0x560, 0x3, 0x2, 0x2, - 0x2, 0x561, 0x562, 0x3, 0x2, 0x2, 0x2, 0x562, 0x565, 0x3, 0x2, 0x2, - 0x2, 0x563, 0x564, 0x7, 0x73, 0x2, 0x2, 0x564, 0x566, 0x9, 0x11, 0x2, - 0x2, 0x565, 0x563, 0x3, 0x2, 0x2, 0x2, 0x565, 0x566, 0x3, 0x2, 0x2, - 0x2, 0x566, 0x569, 0x3, 0x2, 0x2, 0x2, 0x567, 0x568, 0x7, 0x1a, 0x2, - 0x2, 0x568, 0x56a, 0x7, 0xbe, 0x2, 0x2, 0x569, 0x567, 0x3, 0x2, 0x2, - 0x2, 0x569, 0x56a, 0x3, 0x2, 0x2, 0x2, 0x56a, 0x97, 0x3, 0x2, 0x2, 0x2, - 0x56b, 0x56e, 0x5, 0xca, 0x66, 0x2, 0x56c, 0x56d, 0x7, 0xdb, 0x2, 0x2, - 0x56d, 0x56f, 0x5, 0xca, 0x66, 0x2, 0x56e, 0x56c, 0x3, 0x2, 0x2, 0x2, - 0x56e, 0x56f, 0x3, 0x2, 0x2, 0x2, 0x56f, 0x99, 0x3, 0x2, 0x2, 0x2, 0x570, - 0x575, 0x5, 0x9c, 0x4f, 0x2, 0x571, 0x572, 0x7, 0xc4, 0x2, 0x2, 0x572, - 0x574, 0x5, 0x9c, 0x4f, 0x2, 0x573, 0x571, 0x3, 0x2, 0x2, 0x2, 0x574, - 0x577, 0x3, 0x2, 0x2, 0x2, 0x575, 0x573, 0x3, 0x2, 0x2, 0x2, 0x575, - 0x576, 0x3, 0x2, 0x2, 0x2, 0x576, 0x9b, 0x3, 0x2, 0x2, 0x2, 0x577, 0x575, - 0x3, 0x2, 0x2, 0x2, 0x578, 0x579, 0x5, 0xd6, 0x6c, 0x2, 0x579, 0x57a, - 0x7, 0xc9, 0x2, 0x2, 0x57a, 0x57b, 0x5, 0xcc, 0x67, 0x2, 0x57b, 0x9d, - 0x3, 0x2, 0x2, 0x2, 0x57c, 0x57d, 0x7, 0x8f, 0x2, 0x2, 0x57d, 0x57e, - 0x5, 0x9a, 0x4e, 0x2, 0x57e, 0x9f, 0x3, 0x2, 0x2, 0x2, 0x57f, 0x580, - 0x7, 0x91, 0x2, 0x2, 0x580, 0x581, 0x7, 0x1e, 0x2, 0x2, 0x581, 0x582, - 0x7, 0x21, 0x2, 0x2, 0x582, 0x5aa, 0x5, 0xc6, 0x64, 0x2, 0x583, 0x584, - 0x7, 0x91, 0x2, 0x2, 0x584, 0x585, 0x7, 0x1e, 0x2, 0x2, 0x585, 0x586, - 0x7, 0x2e, 0x2, 0x2, 0x586, 0x5aa, 0x5, 0xc0, 0x61, 0x2, 0x587, 0x588, - 0x7, 0x91, 0x2, 0x2, 0x588, 0x58a, 0x7, 0x1e, 0x2, 0x2, 0x589, 0x58b, - 0x7, 0x9b, 0x2, 0x2, 0x58a, 0x589, 0x3, 0x2, 0x2, 0x2, 0x58a, 0x58b, - 0x3, 0x2, 0x2, 0x2, 0x58b, 0x58d, 0x3, 0x2, 0x2, 0x2, 0x58c, 0x58e, - 0x7, 0x99, 0x2, 0x2, 0x58d, 0x58c, 0x3, 0x2, 0x2, 0x2, 0x58d, 0x58e, - 0x3, 0x2, 0x2, 0x2, 0x58e, 0x58f, 0x3, 0x2, 0x2, 0x2, 0x58f, 0x5aa, - 0x5, 0xc0, 0x61, 0x2, 0x590, 0x591, 0x7, 0x91, 0x2, 0x2, 0x591, 0x5aa, - 0x7, 0x22, 0x2, 0x2, 0x592, 0x593, 0x7, 0x91, 0x2, 0x2, 0x593, 0x596, - 0x7, 0x2d, 0x2, 0x2, 0x594, 0x595, 0x7, 0x42, 0x2, 0x2, 0x595, 0x597, - 0x5, 0xc6, 0x64, 0x2, 0x596, 0x594, 0x3, 0x2, 0x2, 0x2, 0x596, 0x597, - 0x3, 0x2, 0x2, 0x2, 0x597, 0x5aa, 0x3, 0x2, 0x2, 0x2, 0x598, 0x59a, - 0x7, 0x91, 0x2, 0x2, 0x599, 0x59b, 0x7, 0x9b, 0x2, 0x2, 0x59a, 0x599, - 0x3, 0x2, 0x2, 0x2, 0x59a, 0x59b, 0x3, 0x2, 0x2, 0x2, 0x59b, 0x59c, - 0x3, 0x2, 0x2, 0x2, 0x59c, 0x59f, 0x7, 0x9a, 0x2, 0x2, 0x59d, 0x59e, - 0x9, 0x12, 0x2, 0x2, 0x59e, 0x5a0, 0x5, 0xc6, 0x64, 0x2, 0x59f, 0x59d, - 0x3, 0x2, 0x2, 0x2, 0x59f, 0x5a0, 0x3, 0x2, 0x2, 0x2, 0x5a0, 0x5a4, - 0x3, 0x2, 0x2, 0x2, 0x5a1, 0x5a2, 0x7, 0x60, 0x2, 0x2, 0x5a2, 0x5a5, - 0x7, 0xbe, 0x2, 0x2, 0x5a3, 0x5a5, 0x5, 0x78, 0x3d, 0x2, 0x5a4, 0x5a1, - 0x3, 0x2, 0x2, 0x2, 0x5a4, 0x5a3, 0x3, 0x2, 0x2, 0x2, 0x5a4, 0x5a5, - 0x3, 0x2, 0x2, 0x2, 0x5a5, 0x5a7, 0x3, 0x2, 0x2, 0x2, 0x5a6, 0x5a8, - 0x5, 0x84, 0x43, 0x2, 0x5a7, 0x5a6, 0x3, 0x2, 0x2, 0x2, 0x5a7, 0x5a8, - 0x3, 0x2, 0x2, 0x2, 0x5a8, 0x5aa, 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x57f, - 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x583, 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x587, - 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x590, 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x592, - 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x598, 0x3, 0x2, 0x2, 0x2, 0x5aa, 0xa1, 0x3, - 0x2, 0x2, 0x2, 0x5ab, 0x5ac, 0x7, 0x98, 0x2, 0x2, 0x5ac, 0x5ad, 0x7, - 0x3e, 0x2, 0x2, 0x5ad, 0x5ae, 0x7, 0x31, 0x2, 0x2, 0x5ae, 0x5ce, 0x5, - 0xc0, 0x61, 0x2, 0x5af, 0x5b0, 0x7, 0x98, 0x2, 0x2, 0x5b0, 0x5b1, 0x7, - 0x3e, 0x2, 0x2, 0x5b1, 0x5ce, 0x7, 0x64, 0x2, 0x2, 0x5b2, 0x5b3, 0x7, - 0x98, 0x2, 0x2, 0x5b3, 0x5b4, 0x7, 0x82, 0x2, 0x2, 0x5b4, 0x5ce, 0x7, - 0x2d, 0x2, 0x2, 0x5b5, 0x5b6, 0x7, 0x98, 0x2, 0x2, 0x5b6, 0x5b7, 0x7, - 0x82, 0x2, 0x2, 0x5b7, 0x5b8, 0x7, 0x2e, 0x2, 0x2, 0x5b8, 0x5ce, 0x5, - 0xc0, 0x61, 0x2, 0x5b9, 0x5ba, 0x7, 0x98, 0x2, 0x2, 0x5ba, 0x5c2, 0x9, - 0x13, 0x2, 0x2, 0x5bb, 0x5bc, 0x7, 0x31, 0x2, 0x2, 0x5bc, 0x5c3, 0x7, - 0x8e, 0x2, 0x2, 0x5bd, 0x5c3, 0x7, 0x3b, 0x2, 0x2, 0x5be, 0x5c0, 0x7, - 0xa7, 0x2, 0x2, 0x5bf, 0x5be, 0x3, 0x2, 0x2, 0x2, 0x5bf, 0x5c0, 0x3, - 0x2, 0x2, 0x2, 0x5c0, 0x5c1, 0x3, 0x2, 0x2, 0x2, 0x5c1, 0x5c3, 0x7, - 0x68, 0x2, 0x2, 0x5c2, 0x5bb, 0x3, 0x2, 0x2, 0x2, 0x5c2, 0x5bd, 0x3, - 0x2, 0x2, 0x2, 0x5c2, 0x5bf, 0x3, 0x2, 0x2, 0x2, 0x5c3, 0x5c4, 0x3, - 0x2, 0x2, 0x2, 0x5c4, 0x5ce, 0x5, 0xc0, 0x61, 0x2, 0x5c5, 0x5c6, 0x7, - 0x98, 0x2, 0x2, 0x5c6, 0x5c7, 0x9, 0x13, 0x2, 0x2, 0x5c7, 0x5c8, 0x7, - 0x87, 0x2, 0x2, 0x5c8, 0x5ce, 0x7, 0x8e, 0x2, 0x2, 0x5c9, 0x5ca, 0x7, - 0x98, 0x2, 0x2, 0x5ca, 0x5cb, 0x7, 0x96, 0x2, 0x2, 0x5cb, 0x5cc, 0x7, - 0x86, 0x2, 0x2, 0x5cc, 0x5ce, 0x5, 0xc0, 0x61, 0x2, 0x5cd, 0x5ab, 0x3, - 0x2, 0x2, 0x2, 0x5cd, 0x5af, 0x3, 0x2, 0x2, 0x2, 0x5cd, 0x5b2, 0x3, - 0x2, 0x2, 0x2, 0x5cd, 0x5b5, 0x3, 0x2, 0x2, 0x2, 0x5cd, 0x5b9, 0x3, - 0x2, 0x2, 0x2, 0x5cd, 0x5c5, 0x3, 0x2, 0x2, 0x2, 0x5cd, 0x5c9, 0x3, - 0x2, 0x2, 0x2, 0x5ce, 0xa3, 0x3, 0x2, 0x2, 0x2, 0x5cf, 0x5d1, 0x7, 0xa6, - 0x2, 0x2, 0x5d0, 0x5d2, 0x7, 0x9b, 0x2, 0x2, 0x5d1, 0x5d0, 0x3, 0x2, - 0x2, 0x2, 0x5d1, 0x5d2, 0x3, 0x2, 0x2, 0x2, 0x5d2, 0x5d4, 0x3, 0x2, - 0x2, 0x2, 0x5d3, 0x5d5, 0x7, 0x99, 0x2, 0x2, 0x5d4, 0x5d3, 0x3, 0x2, - 0x2, 0x2, 0x5d4, 0x5d5, 0x3, 0x2, 0x2, 0x2, 0x5d5, 0x5d8, 0x3, 0x2, - 0x2, 0x2, 0x5d6, 0x5d7, 0x7, 0x4c, 0x2, 0x2, 0x5d7, 0x5d9, 0x7, 0x37, - 0x2, 0x2, 0x5d8, 0x5d6, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5d9, 0x3, 0x2, - 0x2, 0x2, 0x5d9, 0x5da, 0x3, 0x2, 0x2, 0x2, 0x5da, 0x5dc, 0x5, 0xc0, - 0x61, 0x2, 0x5db, 0x5dd, 0x5, 0x2c, 0x17, 0x2, 0x5dc, 0x5db, 0x3, 0x2, - 0x2, 0x2, 0x5dc, 0x5dd, 0x3, 0x2, 0x2, 0x2, 0x5dd, 0xa5, 0x3, 0x2, 0x2, - 0x2, 0x5de, 0x5df, 0x7, 0xab, 0x2, 0x2, 0x5df, 0x5e0, 0x5, 0xc6, 0x64, - 0x2, 0x5e0, 0xa7, 0x3, 0x2, 0x2, 0x2, 0x5e1, 0x5e2, 0x7, 0xb1, 0x2, - 0x2, 0x5e2, 0x5e4, 0x5, 0xc0, 0x61, 0x2, 0x5e3, 0x5e5, 0x7, 0x36, 0x2, - 0x2, 0x5e4, 0x5e3, 0x3, 0x2, 0x2, 0x2, 0x5e4, 0x5e5, 0x3, 0x2, 0x2, - 0x2, 0x5e5, 0x5e8, 0x3, 0x2, 0x2, 0x2, 0x5e6, 0x5e7, 0x7, 0x61, 0x2, - 0x2, 0x5e7, 0x5e9, 0x7, 0xbc, 0x2, 0x2, 0x5e8, 0x5e6, 0x3, 0x2, 0x2, - 0x2, 0x5e8, 0x5e9, 0x3, 0x2, 0x2, 0x2, 0x5e9, 0xa9, 0x3, 0x2, 0x2, 0x2, - 0x5ea, 0x61a, 0x5, 0xd6, 0x6c, 0x2, 0x5eb, 0x5ec, 0x5, 0xd6, 0x6c, 0x2, - 0x5ec, 0x5ed, 0x7, 0xcf, 0x2, 0x2, 0x5ed, 0x5ee, 0x5, 0xd6, 0x6c, 0x2, - 0x5ee, 0x5f5, 0x5, 0xaa, 0x56, 0x2, 0x5ef, 0x5f0, 0x7, 0xc4, 0x2, 0x2, - 0x5f0, 0x5f1, 0x5, 0xd6, 0x6c, 0x2, 0x5f1, 0x5f2, 0x5, 0xaa, 0x56, 0x2, - 0x5f2, 0x5f4, 0x3, 0x2, 0x2, 0x2, 0x5f3, 0x5ef, 0x3, 0x2, 0x2, 0x2, - 0x5f4, 0x5f7, 0x3, 0x2, 0x2, 0x2, 0x5f5, 0x5f3, 0x3, 0x2, 0x2, 0x2, - 0x5f5, 0x5f6, 0x3, 0x2, 0x2, 0x2, 0x5f6, 0x5f8, 0x3, 0x2, 0x2, 0x2, - 0x5f7, 0x5f5, 0x3, 0x2, 0x2, 0x2, 0x5f8, 0x5f9, 0x7, 0xd9, 0x2, 0x2, - 0x5f9, 0x61a, 0x3, 0x2, 0x2, 0x2, 0x5fa, 0x5fb, 0x5, 0xd6, 0x6c, 0x2, - 0x5fb, 0x5fc, 0x7, 0xcf, 0x2, 0x2, 0x5fc, 0x601, 0x5, 0xda, 0x6e, 0x2, - 0x5fd, 0x5fe, 0x7, 0xc4, 0x2, 0x2, 0x5fe, 0x600, 0x5, 0xda, 0x6e, 0x2, - 0x5ff, 0x5fd, 0x3, 0x2, 0x2, 0x2, 0x600, 0x603, 0x3, 0x2, 0x2, 0x2, - 0x601, 0x5ff, 0x3, 0x2, 0x2, 0x2, 0x601, 0x602, 0x3, 0x2, 0x2, 0x2, - 0x602, 0x604, 0x3, 0x2, 0x2, 0x2, 0x603, 0x601, 0x3, 0x2, 0x2, 0x2, - 0x604, 0x605, 0x7, 0xd9, 0x2, 0x2, 0x605, 0x61a, 0x3, 0x2, 0x2, 0x2, - 0x606, 0x607, 0x5, 0xd6, 0x6c, 0x2, 0x607, 0x608, 0x7, 0xcf, 0x2, 0x2, - 0x608, 0x60d, 0x5, 0xaa, 0x56, 0x2, 0x609, 0x60a, 0x7, 0xc4, 0x2, 0x2, - 0x60a, 0x60c, 0x5, 0xaa, 0x56, 0x2, 0x60b, 0x609, 0x3, 0x2, 0x2, 0x2, - 0x60c, 0x60f, 0x3, 0x2, 0x2, 0x2, 0x60d, 0x60b, 0x3, 0x2, 0x2, 0x2, - 0x60d, 0x60e, 0x3, 0x2, 0x2, 0x2, 0x60e, 0x610, 0x3, 0x2, 0x2, 0x2, - 0x60f, 0x60d, 0x3, 0x2, 0x2, 0x2, 0x610, 0x611, 0x7, 0xd9, 0x2, 0x2, - 0x611, 0x61a, 0x3, 0x2, 0x2, 0x2, 0x612, 0x613, 0x5, 0xd6, 0x6c, 0x2, - 0x613, 0x615, 0x7, 0xcf, 0x2, 0x2, 0x614, 0x616, 0x5, 0xac, 0x57, 0x2, - 0x615, 0x614, 0x3, 0x2, 0x2, 0x2, 0x615, 0x616, 0x3, 0x2, 0x2, 0x2, - 0x616, 0x617, 0x3, 0x2, 0x2, 0x2, 0x617, 0x618, 0x7, 0xd9, 0x2, 0x2, - 0x618, 0x61a, 0x3, 0x2, 0x2, 0x2, 0x619, 0x5ea, 0x3, 0x2, 0x2, 0x2, - 0x619, 0x5eb, 0x3, 0x2, 0x2, 0x2, 0x619, 0x5fa, 0x3, 0x2, 0x2, 0x2, - 0x619, 0x606, 0x3, 0x2, 0x2, 0x2, 0x619, 0x612, 0x3, 0x2, 0x2, 0x2, - 0x61a, 0xab, 0x3, 0x2, 0x2, 0x2, 0x61b, 0x620, 0x5, 0xae, 0x58, 0x2, - 0x61c, 0x61d, 0x7, 0xc4, 0x2, 0x2, 0x61d, 0x61f, 0x5, 0xae, 0x58, 0x2, - 0x61e, 0x61c, 0x3, 0x2, 0x2, 0x2, 0x61f, 0x622, 0x3, 0x2, 0x2, 0x2, - 0x620, 0x61e, 0x3, 0x2, 0x2, 0x2, 0x620, 0x621, 0x3, 0x2, 0x2, 0x2, - 0x621, 0xad, 0x3, 0x2, 0x2, 0x2, 0x622, 0x620, 0x3, 0x2, 0x2, 0x2, 0x623, - 0x624, 0x5, 0xc0, 0x61, 0x2, 0x624, 0x625, 0x7, 0xc7, 0x2, 0x2, 0x625, - 0x627, 0x3, 0x2, 0x2, 0x2, 0x626, 0x623, 0x3, 0x2, 0x2, 0x2, 0x626, - 0x627, 0x3, 0x2, 0x2, 0x2, 0x627, 0x628, 0x3, 0x2, 0x2, 0x2, 0x628, - 0x62f, 0x7, 0xc0, 0x2, 0x2, 0x629, 0x62a, 0x7, 0xcf, 0x2, 0x2, 0x62a, - 0x62b, 0x5, 0x68, 0x35, 0x2, 0x62b, 0x62c, 0x7, 0xd9, 0x2, 0x2, 0x62c, - 0x62f, 0x3, 0x2, 0x2, 0x2, 0x62d, 0x62f, 0x5, 0xb0, 0x59, 0x2, 0x62e, - 0x626, 0x3, 0x2, 0x2, 0x2, 0x62e, 0x629, 0x3, 0x2, 0x2, 0x2, 0x62e, - 0x62d, 0x3, 0x2, 0x2, 0x2, 0x62f, 0xaf, 0x3, 0x2, 0x2, 0x2, 0x630, 0x631, - 0x8, 0x59, 0x1, 0x2, 0x631, 0x633, 0x7, 0x14, 0x2, 0x2, 0x632, 0x634, - 0x5, 0xb0, 0x59, 0x2, 0x633, 0x632, 0x3, 0x2, 0x2, 0x2, 0x633, 0x634, - 0x3, 0x2, 0x2, 0x2, 0x634, 0x63a, 0x3, 0x2, 0x2, 0x2, 0x635, 0x636, - 0x7, 0xb3, 0x2, 0x2, 0x636, 0x637, 0x5, 0xb0, 0x59, 0x2, 0x637, 0x638, - 0x7, 0x9d, 0x2, 0x2, 0x638, 0x639, 0x5, 0xb0, 0x59, 0x2, 0x639, 0x63b, - 0x3, 0x2, 0x2, 0x2, 0x63a, 0x635, 0x3, 0x2, 0x2, 0x2, 0x63b, 0x63c, - 0x3, 0x2, 0x2, 0x2, 0x63c, 0x63a, 0x3, 0x2, 0x2, 0x2, 0x63c, 0x63d, - 0x3, 0x2, 0x2, 0x2, 0x63d, 0x640, 0x3, 0x2, 0x2, 0x2, 0x63e, 0x63f, - 0x7, 0x33, 0x2, 0x2, 0x63f, 0x641, 0x5, 0xb0, 0x59, 0x2, 0x640, 0x63e, - 0x3, 0x2, 0x2, 0x2, 0x640, 0x641, 0x3, 0x2, 0x2, 0x2, 0x641, 0x642, - 0x3, 0x2, 0x2, 0x2, 0x642, 0x643, 0x7, 0x34, 0x2, 0x2, 0x643, 0x69c, - 0x3, 0x2, 0x2, 0x2, 0x644, 0x645, 0x7, 0x15, 0x2, 0x2, 0x645, 0x646, - 0x7, 0xcf, 0x2, 0x2, 0x646, 0x647, 0x5, 0xb0, 0x59, 0x2, 0x647, 0x648, - 0x7, 0xc, 0x2, 0x2, 0x648, 0x649, 0x5, 0xaa, 0x56, 0x2, 0x649, 0x64a, - 0x7, 0xd9, 0x2, 0x2, 0x64a, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x64b, 0x64c, - 0x7, 0x23, 0x2, 0x2, 0x64c, 0x69c, 0x7, 0xbe, 0x2, 0x2, 0x64d, 0x64e, - 0x7, 0x3a, 0x2, 0x2, 0x64e, 0x64f, 0x7, 0xcf, 0x2, 0x2, 0x64f, 0x650, - 0x5, 0xce, 0x68, 0x2, 0x650, 0x651, 0x7, 0x42, 0x2, 0x2, 0x651, 0x652, - 0x5, 0xb0, 0x59, 0x2, 0x652, 0x653, 0x7, 0xd9, 0x2, 0x2, 0x653, 0x69c, - 0x3, 0x2, 0x2, 0x2, 0x654, 0x655, 0x7, 0x54, 0x2, 0x2, 0x655, 0x656, - 0x5, 0xb0, 0x59, 0x2, 0x656, 0x657, 0x5, 0xce, 0x68, 0x2, 0x657, 0x69c, - 0x3, 0x2, 0x2, 0x2, 0x658, 0x659, 0x7, 0x95, 0x2, 0x2, 0x659, 0x65a, - 0x7, 0xcf, 0x2, 0x2, 0x65a, 0x65b, 0x5, 0xb0, 0x59, 0x2, 0x65b, 0x65c, - 0x7, 0x42, 0x2, 0x2, 0x65c, 0x65f, 0x5, 0xb0, 0x59, 0x2, 0x65d, 0x65e, - 0x7, 0x3f, 0x2, 0x2, 0x65e, 0x660, 0x5, 0xb0, 0x59, 0x2, 0x65f, 0x65d, - 0x3, 0x2, 0x2, 0x2, 0x65f, 0x660, 0x3, 0x2, 0x2, 0x2, 0x660, 0x661, - 0x3, 0x2, 0x2, 0x2, 0x661, 0x662, 0x7, 0xd9, 0x2, 0x2, 0x662, 0x69c, - 0x3, 0x2, 0x2, 0x2, 0x663, 0x664, 0x7, 0xa0, 0x2, 0x2, 0x664, 0x69c, - 0x7, 0xbe, 0x2, 0x2, 0x665, 0x666, 0x7, 0xa5, 0x2, 0x2, 0x666, 0x667, - 0x7, 0xcf, 0x2, 0x2, 0x667, 0x668, 0x9, 0x14, 0x2, 0x2, 0x668, 0x669, - 0x7, 0xbe, 0x2, 0x2, 0x669, 0x66a, 0x7, 0x42, 0x2, 0x2, 0x66a, 0x66b, - 0x5, 0xb0, 0x59, 0x2, 0x66b, 0x66c, 0x7, 0xd9, 0x2, 0x2, 0x66c, 0x69c, - 0x3, 0x2, 0x2, 0x2, 0x66d, 0x673, 0x5, 0xd6, 0x6c, 0x2, 0x66e, 0x670, - 0x7, 0xcf, 0x2, 0x2, 0x66f, 0x671, 0x5, 0xac, 0x57, 0x2, 0x670, 0x66f, - 0x3, 0x2, 0x2, 0x2, 0x670, 0x671, 0x3, 0x2, 0x2, 0x2, 0x671, 0x672, - 0x3, 0x2, 0x2, 0x2, 0x672, 0x674, 0x7, 0xd9, 0x2, 0x2, 0x673, 0x66e, - 0x3, 0x2, 0x2, 0x2, 0x673, 0x674, 0x3, 0x2, 0x2, 0x2, 0x674, 0x675, - 0x3, 0x2, 0x2, 0x2, 0x675, 0x677, 0x7, 0xcf, 0x2, 0x2, 0x676, 0x678, - 0x7, 0x30, 0x2, 0x2, 0x677, 0x676, 0x3, 0x2, 0x2, 0x2, 0x677, 0x678, - 0x3, 0x2, 0x2, 0x2, 0x678, 0x67a, 0x3, 0x2, 0x2, 0x2, 0x679, 0x67b, - 0x5, 0xb2, 0x5a, 0x2, 0x67a, 0x679, 0x3, 0x2, 0x2, 0x2, 0x67a, 0x67b, - 0x3, 0x2, 0x2, 0x2, 0x67b, 0x67c, 0x3, 0x2, 0x2, 0x2, 0x67c, 0x67d, - 0x7, 0xd9, 0x2, 0x2, 0x67d, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x67e, 0x69c, - 0x5, 0xcc, 0x67, 0x2, 0x67f, 0x680, 0x7, 0xc6, 0x2, 0x2, 0x680, 0x69c, - 0x5, 0xb0, 0x59, 0x13, 0x681, 0x682, 0x7, 0x71, 0x2, 0x2, 0x682, 0x69c, - 0x5, 0xb0, 0x59, 0xe, 0x683, 0x684, 0x5, 0xc0, 0x61, 0x2, 0x684, 0x685, - 0x7, 0xc7, 0x2, 0x2, 0x685, 0x687, 0x3, 0x2, 0x2, 0x2, 0x686, 0x683, - 0x3, 0x2, 0x2, 0x2, 0x686, 0x687, 0x3, 0x2, 0x2, 0x2, 0x687, 0x688, - 0x3, 0x2, 0x2, 0x2, 0x688, 0x69c, 0x7, 0xc0, 0x2, 0x2, 0x689, 0x68a, - 0x7, 0xcf, 0x2, 0x2, 0x68a, 0x68b, 0x5, 0x68, 0x35, 0x2, 0x68b, 0x68c, - 0x7, 0xd9, 0x2, 0x2, 0x68c, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x68d, 0x68e, - 0x7, 0xcf, 0x2, 0x2, 0x68e, 0x68f, 0x5, 0xb0, 0x59, 0x2, 0x68f, 0x690, - 0x7, 0xd9, 0x2, 0x2, 0x690, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x691, 0x692, - 0x7, 0xcf, 0x2, 0x2, 0x692, 0x693, 0x5, 0xac, 0x57, 0x2, 0x693, 0x694, - 0x7, 0xd9, 0x2, 0x2, 0x694, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x695, 0x697, - 0x7, 0xcd, 0x2, 0x2, 0x696, 0x698, 0x5, 0xac, 0x57, 0x2, 0x697, 0x696, - 0x3, 0x2, 0x2, 0x2, 0x697, 0x698, 0x3, 0x2, 0x2, 0x2, 0x698, 0x699, - 0x3, 0x2, 0x2, 0x2, 0x699, 0x69c, 0x7, 0xd8, 0x2, 0x2, 0x69a, 0x69c, - 0x5, 0xb8, 0x5d, 0x2, 0x69b, 0x630, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x644, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x64b, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x64d, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x654, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x658, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x663, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x665, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x66d, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x67e, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x67f, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x681, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x686, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x689, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x68d, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x691, - 0x3, 0x2, 0x2, 0x2, 0x69b, 0x695, 0x3, 0x2, 0x2, 0x2, 0x69b, 0x69a, - 0x3, 0x2, 0x2, 0x2, 0x69c, 0x6e4, 0x3, 0x2, 0x2, 0x2, 0x69d, 0x69e, - 0xc, 0x12, 0x2, 0x2, 0x69e, 0x69f, 0x9, 0x15, 0x2, 0x2, 0x69f, 0x6e3, - 0x5, 0xb0, 0x59, 0x13, 0x6a0, 0x6a1, 0xc, 0x11, 0x2, 0x2, 0x6a1, 0x6a2, - 0x9, 0x16, 0x2, 0x2, 0x6a2, 0x6e3, 0x5, 0xb0, 0x59, 0x12, 0x6a3, 0x6b6, - 0xc, 0x10, 0x2, 0x2, 0x6a4, 0x6b7, 0x7, 0xc8, 0x2, 0x2, 0x6a5, 0x6b7, - 0x7, 0xc9, 0x2, 0x2, 0x6a6, 0x6b7, 0x7, 0xd1, 0x2, 0x2, 0x6a7, 0x6b7, - 0x7, 0xce, 0x2, 0x2, 0x6a8, 0x6b7, 0x7, 0xca, 0x2, 0x2, 0x6a9, 0x6b7, - 0x7, 0xd0, 0x2, 0x2, 0x6aa, 0x6b7, 0x7, 0xcb, 0x2, 0x2, 0x6ab, 0x6ad, - 0x7, 0x45, 0x2, 0x2, 0x6ac, 0x6ab, 0x3, 0x2, 0x2, 0x2, 0x6ac, 0x6ad, - 0x3, 0x2, 0x2, 0x2, 0x6ad, 0x6af, 0x3, 0x2, 0x2, 0x2, 0x6ae, 0x6b0, - 0x7, 0x71, 0x2, 0x2, 0x6af, 0x6ae, 0x3, 0x2, 0x2, 0x2, 0x6af, 0x6b0, - 0x3, 0x2, 0x2, 0x2, 0x6b0, 0x6b1, 0x3, 0x2, 0x2, 0x2, 0x6b1, 0x6b7, - 0x7, 0x4e, 0x2, 0x2, 0x6b2, 0x6b4, 0x7, 0x71, 0x2, 0x2, 0x6b3, 0x6b2, - 0x3, 0x2, 0x2, 0x2, 0x6b3, 0x6b4, 0x3, 0x2, 0x2, 0x2, 0x6b4, 0x6b5, - 0x3, 0x2, 0x2, 0x2, 0x6b5, 0x6b7, 0x9, 0x17, 0x2, 0x2, 0x6b6, 0x6a4, - 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6a5, 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6a6, - 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6a7, 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6a8, - 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6a9, 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6aa, - 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6ac, 0x3, 0x2, 0x2, 0x2, 0x6b6, 0x6b3, - 0x3, 0x2, 0x2, 0x2, 0x6b7, 0x6b8, 0x3, 0x2, 0x2, 0x2, 0x6b8, 0x6e3, - 0x5, 0xb0, 0x59, 0x11, 0x6b9, 0x6ba, 0xc, 0xd, 0x2, 0x2, 0x6ba, 0x6bb, - 0x7, 0x8, 0x2, 0x2, 0x6bb, 0x6e3, 0x5, 0xb0, 0x59, 0xe, 0x6bc, 0x6bd, - 0xc, 0xc, 0x2, 0x2, 0x6bd, 0x6be, 0x7, 0x77, 0x2, 0x2, 0x6be, 0x6e3, - 0x5, 0xb0, 0x59, 0xd, 0x6bf, 0x6c1, 0xc, 0xb, 0x2, 0x2, 0x6c0, 0x6c2, - 0x7, 0x71, 0x2, 0x2, 0x6c1, 0x6c0, 0x3, 0x2, 0x2, 0x2, 0x6c1, 0x6c2, - 0x3, 0x2, 0x2, 0x2, 0x6c2, 0x6c3, 0x3, 0x2, 0x2, 0x2, 0x6c3, 0x6c4, - 0x7, 0x11, 0x2, 0x2, 0x6c4, 0x6c5, 0x5, 0xb0, 0x59, 0x2, 0x6c5, 0x6c6, - 0x7, 0x8, 0x2, 0x2, 0x6c6, 0x6c7, 0x5, 0xb0, 0x59, 0xc, 0x6c7, 0x6e3, - 0x3, 0x2, 0x2, 0x2, 0x6c8, 0x6c9, 0xc, 0xa, 0x2, 0x2, 0x6c9, 0x6ca, - 0x7, 0xd4, 0x2, 0x2, 0x6ca, 0x6cb, 0x5, 0xb0, 0x59, 0x2, 0x6cb, 0x6cc, - 0x7, 0xc3, 0x2, 0x2, 0x6cc, 0x6cd, 0x5, 0xb0, 0x59, 0xa, 0x6cd, 0x6e3, - 0x3, 0x2, 0x2, 0x2, 0x6ce, 0x6cf, 0xc, 0x15, 0x2, 0x2, 0x6cf, 0x6d0, - 0x7, 0xcd, 0x2, 0x2, 0x6d0, 0x6d1, 0x5, 0xb0, 0x59, 0x2, 0x6d1, 0x6d2, - 0x7, 0xd8, 0x2, 0x2, 0x6d2, 0x6e3, 0x3, 0x2, 0x2, 0x2, 0x6d3, 0x6d4, - 0xc, 0x14, 0x2, 0x2, 0x6d4, 0x6d5, 0x7, 0xc7, 0x2, 0x2, 0x6d5, 0x6e3, - 0x7, 0xbc, 0x2, 0x2, 0x6d6, 0x6d7, 0xc, 0xf, 0x2, 0x2, 0x6d7, 0x6d9, - 0x7, 0x56, 0x2, 0x2, 0x6d8, 0x6da, 0x7, 0x71, 0x2, 0x2, 0x6d9, 0x6d8, - 0x3, 0x2, 0x2, 0x2, 0x6d9, 0x6da, 0x3, 0x2, 0x2, 0x2, 0x6da, 0x6db, - 0x3, 0x2, 0x2, 0x2, 0x6db, 0x6e3, 0x7, 0x72, 0x2, 0x2, 0x6dc, 0x6e0, - 0xc, 0x9, 0x2, 0x2, 0x6dd, 0x6e1, 0x5, 0xd4, 0x6b, 0x2, 0x6de, 0x6df, - 0x7, 0xc, 0x2, 0x2, 0x6df, 0x6e1, 0x5, 0xd6, 0x6c, 0x2, 0x6e0, 0x6dd, - 0x3, 0x2, 0x2, 0x2, 0x6e0, 0x6de, 0x3, 0x2, 0x2, 0x2, 0x6e1, 0x6e3, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x69d, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6a0, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6a3, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6b9, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6bc, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6bf, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6c8, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6ce, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6d3, 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6d6, - 0x3, 0x2, 0x2, 0x2, 0x6e2, 0x6dc, 0x3, 0x2, 0x2, 0x2, 0x6e3, 0x6e6, - 0x3, 0x2, 0x2, 0x2, 0x6e4, 0x6e2, 0x3, 0x2, 0x2, 0x2, 0x6e4, 0x6e5, - 0x3, 0x2, 0x2, 0x2, 0x6e5, 0xb1, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6e4, 0x3, - 0x2, 0x2, 0x2, 0x6e7, 0x6ec, 0x5, 0xb4, 0x5b, 0x2, 0x6e8, 0x6e9, 0x7, - 0xc4, 0x2, 0x2, 0x6e9, 0x6eb, 0x5, 0xb4, 0x5b, 0x2, 0x6ea, 0x6e8, 0x3, - 0x2, 0x2, 0x2, 0x6eb, 0x6ee, 0x3, 0x2, 0x2, 0x2, 0x6ec, 0x6ea, 0x3, - 0x2, 0x2, 0x2, 0x6ec, 0x6ed, 0x3, 0x2, 0x2, 0x2, 0x6ed, 0xb3, 0x3, 0x2, - 0x2, 0x2, 0x6ee, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6ef, 0x6f2, 0x5, 0xb6, - 0x5c, 0x2, 0x6f0, 0x6f2, 0x5, 0xb0, 0x59, 0x2, 0x6f1, 0x6ef, 0x3, 0x2, - 0x2, 0x2, 0x6f1, 0x6f0, 0x3, 0x2, 0x2, 0x2, 0x6f2, 0xb5, 0x3, 0x2, 0x2, - 0x2, 0x6f3, 0x6f4, 0x7, 0xcf, 0x2, 0x2, 0x6f4, 0x6f9, 0x5, 0xd6, 0x6c, - 0x2, 0x6f5, 0x6f6, 0x7, 0xc4, 0x2, 0x2, 0x6f6, 0x6f8, 0x5, 0xd6, 0x6c, - 0x2, 0x6f7, 0x6f5, 0x3, 0x2, 0x2, 0x2, 0x6f8, 0x6fb, 0x3, 0x2, 0x2, - 0x2, 0x6f9, 0x6f7, 0x3, 0x2, 0x2, 0x2, 0x6f9, 0x6fa, 0x3, 0x2, 0x2, - 0x2, 0x6fa, 0x6fc, 0x3, 0x2, 0x2, 0x2, 0x6fb, 0x6f9, 0x3, 0x2, 0x2, - 0x2, 0x6fc, 0x6fd, 0x7, 0xd9, 0x2, 0x2, 0x6fd, 0x707, 0x3, 0x2, 0x2, - 0x2, 0x6fe, 0x703, 0x5, 0xd6, 0x6c, 0x2, 0x6ff, 0x700, 0x7, 0xc4, 0x2, - 0x2, 0x700, 0x702, 0x5, 0xd6, 0x6c, 0x2, 0x701, 0x6ff, 0x3, 0x2, 0x2, - 0x2, 0x702, 0x705, 0x3, 0x2, 0x2, 0x2, 0x703, 0x701, 0x3, 0x2, 0x2, - 0x2, 0x703, 0x704, 0x3, 0x2, 0x2, 0x2, 0x704, 0x707, 0x3, 0x2, 0x2, - 0x2, 0x705, 0x703, 0x3, 0x2, 0x2, 0x2, 0x706, 0x6f3, 0x3, 0x2, 0x2, - 0x2, 0x706, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x707, 0x708, 0x3, 0x2, 0x2, - 0x2, 0x708, 0x709, 0x7, 0xbf, 0x2, 0x2, 0x709, 0x70a, 0x5, 0xb0, 0x59, - 0x2, 0x70a, 0xb7, 0x3, 0x2, 0x2, 0x2, 0x70b, 0x70c, 0x5, 0xc0, 0x61, - 0x2, 0x70c, 0x70d, 0x7, 0xc7, 0x2, 0x2, 0x70d, 0x70f, 0x3, 0x2, 0x2, - 0x2, 0x70e, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x70e, 0x70f, 0x3, 0x2, 0x2, - 0x2, 0x70f, 0x710, 0x3, 0x2, 0x2, 0x2, 0x710, 0x711, 0x5, 0xba, 0x5e, - 0x2, 0x711, 0xb9, 0x3, 0x2, 0x2, 0x2, 0x712, 0x715, 0x5, 0xd6, 0x6c, - 0x2, 0x713, 0x714, 0x7, 0xc7, 0x2, 0x2, 0x714, 0x716, 0x5, 0xd6, 0x6c, - 0x2, 0x715, 0x713, 0x3, 0x2, 0x2, 0x2, 0x715, 0x716, 0x3, 0x2, 0x2, - 0x2, 0x716, 0xbb, 0x3, 0x2, 0x2, 0x2, 0x717, 0x718, 0x8, 0x5f, 0x1, - 0x2, 0x718, 0x71f, 0x5, 0xc0, 0x61, 0x2, 0x719, 0x71f, 0x5, 0xbe, 0x60, - 0x2, 0x71a, 0x71b, 0x7, 0xcf, 0x2, 0x2, 0x71b, 0x71c, 0x5, 0x68, 0x35, - 0x2, 0x71c, 0x71d, 0x7, 0xd9, 0x2, 0x2, 0x71d, 0x71f, 0x3, 0x2, 0x2, - 0x2, 0x71e, 0x717, 0x3, 0x2, 0x2, 0x2, 0x71e, 0x719, 0x3, 0x2, 0x2, - 0x2, 0x71e, 0x71a, 0x3, 0x2, 0x2, 0x2, 0x71f, 0x728, 0x3, 0x2, 0x2, - 0x2, 0x720, 0x724, 0xc, 0x3, 0x2, 0x2, 0x721, 0x725, 0x5, 0xd4, 0x6b, - 0x2, 0x722, 0x723, 0x7, 0xc, 0x2, 0x2, 0x723, 0x725, 0x5, 0xd6, 0x6c, - 0x2, 0x724, 0x721, 0x3, 0x2, 0x2, 0x2, 0x724, 0x722, 0x3, 0x2, 0x2, - 0x2, 0x725, 0x727, 0x3, 0x2, 0x2, 0x2, 0x726, 0x720, 0x3, 0x2, 0x2, - 0x2, 0x727, 0x72a, 0x3, 0x2, 0x2, 0x2, 0x728, 0x726, 0x3, 0x2, 0x2, - 0x2, 0x728, 0x729, 0x3, 0x2, 0x2, 0x2, 0x729, 0xbd, 0x3, 0x2, 0x2, 0x2, - 0x72a, 0x728, 0x3, 0x2, 0x2, 0x2, 0x72b, 0x72c, 0x5, 0xd6, 0x6c, 0x2, - 0x72c, 0x72e, 0x7, 0xcf, 0x2, 0x2, 0x72d, 0x72f, 0x5, 0xc2, 0x62, 0x2, - 0x72e, 0x72d, 0x3, 0x2, 0x2, 0x2, 0x72e, 0x72f, 0x3, 0x2, 0x2, 0x2, - 0x72f, 0x730, 0x3, 0x2, 0x2, 0x2, 0x730, 0x731, 0x7, 0xd9, 0x2, 0x2, - 0x731, 0xbf, 0x3, 0x2, 0x2, 0x2, 0x732, 0x733, 0x5, 0xc6, 0x64, 0x2, - 0x733, 0x734, 0x7, 0xc7, 0x2, 0x2, 0x734, 0x736, 0x3, 0x2, 0x2, 0x2, - 0x735, 0x732, 0x3, 0x2, 0x2, 0x2, 0x735, 0x736, 0x3, 0x2, 0x2, 0x2, - 0x736, 0x737, 0x3, 0x2, 0x2, 0x2, 0x737, 0x738, 0x5, 0xd6, 0x6c, 0x2, - 0x738, 0xc1, 0x3, 0x2, 0x2, 0x2, 0x739, 0x73e, 0x5, 0xc4, 0x63, 0x2, - 0x73a, 0x73b, 0x7, 0xc4, 0x2, 0x2, 0x73b, 0x73d, 0x5, 0xc4, 0x63, 0x2, - 0x73c, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x73d, 0x740, 0x3, 0x2, 0x2, 0x2, - 0x73e, 0x73c, 0x3, 0x2, 0x2, 0x2, 0x73e, 0x73f, 0x3, 0x2, 0x2, 0x2, - 0x73f, 0xc3, 0x3, 0x2, 0x2, 0x2, 0x740, 0x73e, 0x3, 0x2, 0x2, 0x2, 0x741, - 0x745, 0x5, 0xc0, 0x61, 0x2, 0x742, 0x745, 0x5, 0xbe, 0x60, 0x2, 0x743, - 0x745, 0x5, 0xcc, 0x67, 0x2, 0x744, 0x741, 0x3, 0x2, 0x2, 0x2, 0x744, - 0x742, 0x3, 0x2, 0x2, 0x2, 0x744, 0x743, 0x3, 0x2, 0x2, 0x2, 0x745, - 0xc5, 0x3, 0x2, 0x2, 0x2, 0x746, 0x747, 0x5, 0xd6, 0x6c, 0x2, 0x747, - 0xc7, 0x3, 0x2, 0x2, 0x2, 0x748, 0x751, 0x7, 0xba, 0x2, 0x2, 0x749, - 0x74a, 0x7, 0xc7, 0x2, 0x2, 0x74a, 0x751, 0x9, 0x18, 0x2, 0x2, 0x74b, - 0x74c, 0x7, 0xbc, 0x2, 0x2, 0x74c, 0x74e, 0x7, 0xc7, 0x2, 0x2, 0x74d, - 0x74f, 0x9, 0x18, 0x2, 0x2, 0x74e, 0x74d, 0x3, 0x2, 0x2, 0x2, 0x74e, - 0x74f, 0x3, 0x2, 0x2, 0x2, 0x74f, 0x751, 0x3, 0x2, 0x2, 0x2, 0x750, - 0x748, 0x3, 0x2, 0x2, 0x2, 0x750, 0x749, 0x3, 0x2, 0x2, 0x2, 0x750, - 0x74b, 0x3, 0x2, 0x2, 0x2, 0x751, 0xc9, 0x3, 0x2, 0x2, 0x2, 0x752, 0x754, - 0x9, 0x19, 0x2, 0x2, 0x753, 0x752, 0x3, 0x2, 0x2, 0x2, 0x753, 0x754, - 0x3, 0x2, 0x2, 0x2, 0x754, 0x75b, 0x3, 0x2, 0x2, 0x2, 0x755, 0x75c, - 0x5, 0xc8, 0x65, 0x2, 0x756, 0x75c, 0x7, 0xbb, 0x2, 0x2, 0x757, 0x75c, - 0x7, 0xbc, 0x2, 0x2, 0x758, 0x75c, 0x7, 0xbd, 0x2, 0x2, 0x759, 0x75c, - 0x7, 0x50, 0x2, 0x2, 0x75a, 0x75c, 0x7, 0x6f, 0x2, 0x2, 0x75b, 0x755, - 0x3, 0x2, 0x2, 0x2, 0x75b, 0x756, 0x3, 0x2, 0x2, 0x2, 0x75b, 0x757, - 0x3, 0x2, 0x2, 0x2, 0x75b, 0x758, 0x3, 0x2, 0x2, 0x2, 0x75b, 0x759, - 0x3, 0x2, 0x2, 0x2, 0x75b, 0x75a, 0x3, 0x2, 0x2, 0x2, 0x75c, 0xcb, 0x3, - 0x2, 0x2, 0x2, 0x75d, 0x761, 0x5, 0xca, 0x66, 0x2, 0x75e, 0x761, 0x7, - 0xbe, 0x2, 0x2, 0x75f, 0x761, 0x7, 0x72, 0x2, 0x2, 0x760, 0x75d, 0x3, - 0x2, 0x2, 0x2, 0x760, 0x75e, 0x3, 0x2, 0x2, 0x2, 0x760, 0x75f, 0x3, - 0x2, 0x2, 0x2, 0x761, 0xcd, 0x3, 0x2, 0x2, 0x2, 0x762, 0x763, 0x9, 0x1a, - 0x2, 0x2, 0x763, 0xcf, 0x3, 0x2, 0x2, 0x2, 0x764, 0x765, 0x9, 0x1b, - 0x2, 0x2, 0x765, 0xd1, 0x3, 0x2, 0x2, 0x2, 0x766, 0x767, 0x9, 0x1c, - 0x2, 0x2, 0x767, 0xd3, 0x3, 0x2, 0x2, 0x2, 0x768, 0x76b, 0x7, 0xb9, - 0x2, 0x2, 0x769, 0x76b, 0x5, 0xd2, 0x6a, 0x2, 0x76a, 0x768, 0x3, 0x2, - 0x2, 0x2, 0x76a, 0x769, 0x3, 0x2, 0x2, 0x2, 0x76b, 0xd5, 0x3, 0x2, 0x2, - 0x2, 0x76c, 0x770, 0x7, 0xb9, 0x2, 0x2, 0x76d, 0x770, 0x5, 0xce, 0x68, - 0x2, 0x76e, 0x770, 0x5, 0xd0, 0x69, 0x2, 0x76f, 0x76c, 0x3, 0x2, 0x2, - 0x2, 0x76f, 0x76d, 0x3, 0x2, 0x2, 0x2, 0x76f, 0x76e, 0x3, 0x2, 0x2, - 0x2, 0x770, 0xd7, 0x3, 0x2, 0x2, 0x2, 0x771, 0x774, 0x5, 0xd6, 0x6c, - 0x2, 0x772, 0x774, 0x7, 0x72, 0x2, 0x2, 0x773, 0x771, 0x3, 0x2, 0x2, - 0x2, 0x773, 0x772, 0x3, 0x2, 0x2, 0x2, 0x774, 0xd9, 0x3, 0x2, 0x2, 0x2, - 0x775, 0x776, 0x7, 0xbe, 0x2, 0x2, 0x776, 0x777, 0x7, 0xc9, 0x2, 0x2, - 0x777, 0x778, 0x5, 0xca, 0x66, 0x2, 0x778, 0xdb, 0x3, 0x2, 0x2, 0x2, - 0x102, 0xe0, 0xe4, 0xe7, 0xea, 0xfe, 0x104, 0x10b, 0x113, 0x118, 0x11f, - 0x124, 0x12b, 0x130, 0x136, 0x13c, 0x141, 0x147, 0x14c, 0x152, 0x157, - 0x15d, 0x16b, 0x172, 0x179, 0x180, 0x186, 0x18b, 0x191, 0x196, 0x19c, - 0x1a5, 0x1af, 0x1b9, 0x1cd, 0x1d5, 0x1e4, 0x1eb, 0x1f9, 0x1ff, 0x205, - 0x20c, 0x210, 0x213, 0x21a, 0x21e, 0x221, 0x22c, 0x230, 0x233, 0x238, - 0x23a, 0x23d, 0x240, 0x24a, 0x24e, 0x251, 0x254, 0x259, 0x25b, 0x261, - 0x267, 0x26b, 0x26e, 0x271, 0x274, 0x277, 0x27c, 0x282, 0x286, 0x289, - 0x28c, 0x290, 0x298, 0x2b2, 0x2b4, 0x2b8, 0x2ce, 0x2d0, 0x2db, 0x2de, - 0x2e7, 0x2f8, 0x303, 0x315, 0x322, 0x333, 0x33c, 0x357, 0x359, 0x36e, - 0x373, 0x378, 0x37b, 0x387, 0x38c, 0x390, 0x393, 0x397, 0x39b, 0x3a0, - 0x3a3, 0x3a7, 0x3a9, 0x3bf, 0x3c7, 0x3ca, 0x3d4, 0x3d8, 0x3e0, 0x3e4, - 0x3e9, 0x3ed, 0x3f1, 0x3f5, 0x3f9, 0x3fb, 0x403, 0x407, 0x40a, 0x413, - 0x418, 0x41b, 0x425, 0x42f, 0x433, 0x438, 0x43c, 0x442, 0x445, 0x448, - 0x44b, 0x459, 0x45d, 0x461, 0x466, 0x469, 0x473, 0x47b, 0x47e, 0x482, - 0x485, 0x489, 0x48c, 0x48f, 0x492, 0x495, 0x499, 0x49d, 0x4a0, 0x4a3, - 0x4a6, 0x4a9, 0x4ac, 0x4b5, 0x4bb, 0x4cf, 0x4e5, 0x4ed, 0x4f0, 0x4f6, - 0x4fe, 0x501, 0x507, 0x509, 0x50d, 0x512, 0x515, 0x518, 0x51c, 0x520, - 0x523, 0x525, 0x528, 0x52c, 0x530, 0x533, 0x535, 0x537, 0x53a, 0x53f, - 0x54a, 0x550, 0x555, 0x55c, 0x561, 0x565, 0x569, 0x56e, 0x575, 0x58a, - 0x58d, 0x596, 0x59a, 0x59f, 0x5a4, 0x5a7, 0x5a9, 0x5bf, 0x5c2, 0x5cd, - 0x5d1, 0x5d4, 0x5d8, 0x5dc, 0x5e4, 0x5e8, 0x5f5, 0x601, 0x60d, 0x615, - 0x619, 0x620, 0x626, 0x62e, 0x633, 0x63c, 0x640, 0x65f, 0x670, 0x673, - 0x677, 0x67a, 0x686, 0x697, 0x69b, 0x6ac, 0x6af, 0x6b3, 0x6b6, 0x6c1, - 0x6d9, 0x6e0, 0x6e2, 0x6e4, 0x6ec, 0x6f1, 0x6f9, 0x703, 0x706, 0x70e, - 0x715, 0x71e, 0x724, 0x728, 0x72e, 0x735, 0x73e, 0x744, 0x74e, 0x750, - 0x753, 0x75b, 0x760, 0x76a, 0x76f, 0x773, + 0x2, 0x2, 0x2, 0x40b, 0x57, 0x3, 0x2, 0x2, 0x2, 0x40c, 0x40d, 0x7, 0x39, + 0x2, 0x2, 0x40d, 0x40e, 0x7, 0xf, 0x2, 0x2, 0x40e, 0x413, 0x5, 0x4, + 0x3, 0x2, 0x40f, 0x410, 0x7, 0x39, 0x2, 0x2, 0x410, 0x411, 0x7, 0x98, + 0x2, 0x2, 0x411, 0x413, 0x5, 0x4, 0x3, 0x2, 0x412, 0x40c, 0x3, 0x2, + 0x2, 0x2, 0x412, 0x40f, 0x3, 0x2, 0x2, 0x2, 0x413, 0x59, 0x3, 0x2, 0x2, + 0x2, 0x414, 0x415, 0x7, 0x54, 0x2, 0x2, 0x415, 0x417, 0x7, 0x56, 0x2, + 0x2, 0x416, 0x418, 0x7, 0x9a, 0x2, 0x2, 0x417, 0x416, 0x3, 0x2, 0x2, + 0x2, 0x417, 0x418, 0x3, 0x2, 0x2, 0x2, 0x418, 0x41c, 0x3, 0x2, 0x2, + 0x2, 0x419, 0x41d, 0x5, 0xc0, 0x61, 0x2, 0x41a, 0x41b, 0x7, 0x45, 0x2, + 0x2, 0x41b, 0x41d, 0x5, 0xbe, 0x60, 0x2, 0x41c, 0x419, 0x3, 0x2, 0x2, + 0x2, 0x41c, 0x41a, 0x3, 0x2, 0x2, 0x2, 0x41d, 0x41f, 0x3, 0x2, 0x2, + 0x2, 0x41e, 0x420, 0x5, 0x5c, 0x2f, 0x2, 0x41f, 0x41e, 0x3, 0x2, 0x2, + 0x2, 0x41f, 0x420, 0x3, 0x2, 0x2, 0x2, 0x420, 0x421, 0x3, 0x2, 0x2, + 0x2, 0x421, 0x422, 0x5, 0x5e, 0x30, 0x2, 0x422, 0x5b, 0x3, 0x2, 0x2, + 0x2, 0x423, 0x424, 0x7, 0xd0, 0x2, 0x2, 0x424, 0x429, 0x5, 0xba, 0x5e, + 0x2, 0x425, 0x426, 0x7, 0xc5, 0x2, 0x2, 0x426, 0x428, 0x5, 0xba, 0x5e, + 0x2, 0x427, 0x425, 0x3, 0x2, 0x2, 0x2, 0x428, 0x42b, 0x3, 0x2, 0x2, + 0x2, 0x429, 0x427, 0x3, 0x2, 0x2, 0x2, 0x429, 0x42a, 0x3, 0x2, 0x2, + 0x2, 0x42a, 0x42c, 0x3, 0x2, 0x2, 0x2, 0x42b, 0x429, 0x3, 0x2, 0x2, + 0x2, 0x42c, 0x42d, 0x7, 0xda, 0x2, 0x2, 0x42d, 0x5d, 0x3, 0x2, 0x2, + 0x2, 0x42e, 0x42f, 0x7, 0x41, 0x2, 0x2, 0x42f, 0x438, 0x5, 0xd6, 0x6c, + 0x2, 0x430, 0x438, 0x7, 0xaf, 0x2, 0x2, 0x431, 0x433, 0x5, 0x68, 0x35, + 0x2, 0x432, 0x434, 0x7, 0xdb, 0x2, 0x2, 0x433, 0x432, 0x3, 0x2, 0x2, + 0x2, 0x433, 0x434, 0x3, 0x2, 0x2, 0x2, 0x434, 0x435, 0x3, 0x2, 0x2, + 0x2, 0x435, 0x436, 0x7, 0x2, 0x2, 0x3, 0x436, 0x438, 0x3, 0x2, 0x2, + 0x2, 0x437, 0x42e, 0x3, 0x2, 0x2, 0x2, 0x437, 0x430, 0x3, 0x2, 0x2, + 0x2, 0x437, 0x431, 0x3, 0x2, 0x2, 0x2, 0x438, 0x5f, 0x3, 0x2, 0x2, 0x2, + 0x439, 0x43a, 0x7, 0x5b, 0x2, 0x2, 0x43a, 0x43c, 0x7, 0x6f, 0x2, 0x2, + 0x43b, 0x43d, 0x5, 0x2c, 0x17, 0x2, 0x43c, 0x43b, 0x3, 0x2, 0x2, 0x2, + 0x43c, 0x43d, 0x3, 0x2, 0x2, 0x2, 0x43d, 0x43e, 0x3, 0x2, 0x2, 0x2, + 0x43e, 0x440, 0x5, 0x78, 0x3d, 0x2, 0x43f, 0x441, 0x9, 0x7, 0x2, 0x2, + 0x440, 0x43f, 0x3, 0x2, 0x2, 0x2, 0x440, 0x441, 0x3, 0x2, 0x2, 0x2, + 0x441, 0x61, 0x3, 0x2, 0x2, 0x2, 0x442, 0x443, 0x7, 0x77, 0x2, 0x2, + 0x443, 0x444, 0x7, 0x9a, 0x2, 0x2, 0x444, 0x446, 0x5, 0xc0, 0x61, 0x2, + 0x445, 0x447, 0x5, 0x2c, 0x17, 0x2, 0x446, 0x445, 0x3, 0x2, 0x2, 0x2, + 0x446, 0x447, 0x3, 0x2, 0x2, 0x2, 0x447, 0x449, 0x3, 0x2, 0x2, 0x2, + 0x448, 0x44a, 0x5, 0x10, 0x9, 0x2, 0x449, 0x448, 0x3, 0x2, 0x2, 0x2, + 0x449, 0x44a, 0x3, 0x2, 0x2, 0x2, 0x44a, 0x44c, 0x3, 0x2, 0x2, 0x2, + 0x44b, 0x44d, 0x7, 0x3d, 0x2, 0x2, 0x44c, 0x44b, 0x3, 0x2, 0x2, 0x2, + 0x44c, 0x44d, 0x3, 0x2, 0x2, 0x2, 0x44d, 0x44f, 0x3, 0x2, 0x2, 0x2, + 0x44e, 0x450, 0x7, 0x26, 0x2, 0x2, 0x44f, 0x44e, 0x3, 0x2, 0x2, 0x2, + 0x44f, 0x450, 0x3, 0x2, 0x2, 0x2, 0x450, 0x63, 0x3, 0x2, 0x2, 0x2, 0x451, + 0x452, 0x7, 0x85, 0x2, 0x2, 0x452, 0x453, 0x7, 0x9a, 0x2, 0x2, 0x453, + 0x454, 0x5, 0xc0, 0x61, 0x2, 0x454, 0x455, 0x7, 0xa2, 0x2, 0x2, 0x455, + 0x45d, 0x5, 0xc0, 0x61, 0x2, 0x456, 0x457, 0x7, 0xc5, 0x2, 0x2, 0x457, + 0x458, 0x5, 0xc0, 0x61, 0x2, 0x458, 0x459, 0x7, 0xa2, 0x2, 0x2, 0x459, + 0x45a, 0x5, 0xc0, 0x61, 0x2, 0x45a, 0x45c, 0x3, 0x2, 0x2, 0x2, 0x45b, + 0x456, 0x3, 0x2, 0x2, 0x2, 0x45c, 0x45f, 0x3, 0x2, 0x2, 0x2, 0x45d, + 0x45b, 0x3, 0x2, 0x2, 0x2, 0x45d, 0x45e, 0x3, 0x2, 0x2, 0x2, 0x45e, + 0x461, 0x3, 0x2, 0x2, 0x2, 0x45f, 0x45d, 0x3, 0x2, 0x2, 0x2, 0x460, + 0x462, 0x5, 0x2c, 0x17, 0x2, 0x461, 0x460, 0x3, 0x2, 0x2, 0x2, 0x461, + 0x462, 0x3, 0x2, 0x2, 0x2, 0x462, 0x65, 0x3, 0x2, 0x2, 0x2, 0x463, 0x465, + 0x7, 0xd0, 0x2, 0x2, 0x464, 0x466, 0x5, 0x6e, 0x38, 0x2, 0x465, 0x464, + 0x3, 0x2, 0x2, 0x2, 0x465, 0x466, 0x3, 0x2, 0x2, 0x2, 0x466, 0x467, + 0x3, 0x2, 0x2, 0x2, 0x467, 0x468, 0x7, 0x8d, 0x2, 0x2, 0x468, 0x46a, + 0x5, 0xac, 0x57, 0x2, 0x469, 0x46b, 0x5, 0x7a, 0x3e, 0x2, 0x46a, 0x469, + 0x3, 0x2, 0x2, 0x2, 0x46a, 0x46b, 0x3, 0x2, 0x2, 0x2, 0x46b, 0x46d, + 0x3, 0x2, 0x2, 0x2, 0x46c, 0x46e, 0x5, 0x80, 0x41, 0x2, 0x46d, 0x46c, + 0x3, 0x2, 0x2, 0x2, 0x46d, 0x46e, 0x3, 0x2, 0x2, 0x2, 0x46e, 0x46f, + 0x3, 0x2, 0x2, 0x2, 0x46f, 0x470, 0x7, 0xda, 0x2, 0x2, 0x470, 0x67, + 0x3, 0x2, 0x2, 0x2, 0x471, 0x477, 0x5, 0x6a, 0x36, 0x2, 0x472, 0x473, + 0x7, 0xaa, 0x2, 0x2, 0x473, 0x474, 0x7, 0x6, 0x2, 0x2, 0x474, 0x476, + 0x5, 0x6a, 0x36, 0x2, 0x475, 0x472, 0x3, 0x2, 0x2, 0x2, 0x476, 0x479, + 0x3, 0x2, 0x2, 0x2, 0x477, 0x475, 0x3, 0x2, 0x2, 0x2, 0x477, 0x478, + 0x3, 0x2, 0x2, 0x2, 0x478, 0x69, 0x3, 0x2, 0x2, 0x2, 0x479, 0x477, 0x3, + 0x2, 0x2, 0x2, 0x47a, 0x480, 0x5, 0x6c, 0x37, 0x2, 0x47b, 0x47c, 0x7, + 0xd0, 0x2, 0x2, 0x47c, 0x47d, 0x5, 0x68, 0x35, 0x2, 0x47d, 0x47e, 0x7, + 0xda, 0x2, 0x2, 0x47e, 0x480, 0x3, 0x2, 0x2, 0x2, 0x47f, 0x47a, 0x3, + 0x2, 0x2, 0x2, 0x47f, 0x47b, 0x3, 0x2, 0x2, 0x2, 0x480, 0x6b, 0x3, 0x2, + 0x2, 0x2, 0x481, 0x483, 0x5, 0x6e, 0x38, 0x2, 0x482, 0x481, 0x3, 0x2, + 0x2, 0x2, 0x482, 0x483, 0x3, 0x2, 0x2, 0x2, 0x483, 0x484, 0x3, 0x2, + 0x2, 0x2, 0x484, 0x486, 0x7, 0x8d, 0x2, 0x2, 0x485, 0x487, 0x7, 0x31, + 0x2, 0x2, 0x486, 0x485, 0x3, 0x2, 0x2, 0x2, 0x486, 0x487, 0x3, 0x2, + 0x2, 0x2, 0x487, 0x489, 0x3, 0x2, 0x2, 0x2, 0x488, 0x48a, 0x5, 0x70, + 0x39, 0x2, 0x489, 0x488, 0x3, 0x2, 0x2, 0x2, 0x489, 0x48a, 0x3, 0x2, + 0x2, 0x2, 0x48a, 0x48b, 0x3, 0x2, 0x2, 0x2, 0x48b, 0x48d, 0x5, 0xac, + 0x57, 0x2, 0x48c, 0x48e, 0x5, 0x72, 0x3a, 0x2, 0x48d, 0x48c, 0x3, 0x2, + 0x2, 0x2, 0x48d, 0x48e, 0x3, 0x2, 0x2, 0x2, 0x48e, 0x490, 0x3, 0x2, + 0x2, 0x2, 0x48f, 0x491, 0x5, 0x74, 0x3b, 0x2, 0x490, 0x48f, 0x3, 0x2, + 0x2, 0x2, 0x490, 0x491, 0x3, 0x2, 0x2, 0x2, 0x491, 0x493, 0x3, 0x2, + 0x2, 0x2, 0x492, 0x494, 0x5, 0x76, 0x3c, 0x2, 0x493, 0x492, 0x3, 0x2, + 0x2, 0x2, 0x493, 0x494, 0x3, 0x2, 0x2, 0x2, 0x494, 0x496, 0x3, 0x2, + 0x2, 0x2, 0x495, 0x497, 0x5, 0x78, 0x3d, 0x2, 0x496, 0x495, 0x3, 0x2, + 0x2, 0x2, 0x496, 0x497, 0x3, 0x2, 0x2, 0x2, 0x497, 0x499, 0x3, 0x2, + 0x2, 0x2, 0x498, 0x49a, 0x5, 0x7a, 0x3e, 0x2, 0x499, 0x498, 0x3, 0x2, + 0x2, 0x2, 0x499, 0x49a, 0x3, 0x2, 0x2, 0x2, 0x49a, 0x49d, 0x3, 0x2, + 0x2, 0x2, 0x49b, 0x49c, 0x7, 0xb6, 0x2, 0x2, 0x49c, 0x49e, 0x9, 0x8, + 0x2, 0x2, 0x49d, 0x49b, 0x3, 0x2, 0x2, 0x2, 0x49d, 0x49e, 0x3, 0x2, + 0x2, 0x2, 0x49e, 0x4a1, 0x3, 0x2, 0x2, 0x2, 0x49f, 0x4a0, 0x7, 0xb6, + 0x2, 0x2, 0x4a0, 0x4a2, 0x7, 0xa4, 0x2, 0x2, 0x4a1, 0x49f, 0x3, 0x2, + 0x2, 0x2, 0x4a1, 0x4a2, 0x3, 0x2, 0x2, 0x2, 0x4a2, 0x4a4, 0x3, 0x2, + 0x2, 0x2, 0x4a3, 0x4a5, 0x5, 0x7c, 0x3f, 0x2, 0x4a4, 0x4a3, 0x3, 0x2, + 0x2, 0x2, 0x4a4, 0x4a5, 0x3, 0x2, 0x2, 0x2, 0x4a5, 0x4a7, 0x3, 0x2, + 0x2, 0x2, 0x4a6, 0x4a8, 0x5, 0x7e, 0x40, 0x2, 0x4a7, 0x4a6, 0x3, 0x2, + 0x2, 0x2, 0x4a7, 0x4a8, 0x3, 0x2, 0x2, 0x2, 0x4a8, 0x4aa, 0x3, 0x2, + 0x2, 0x2, 0x4a9, 0x4ab, 0x5, 0x82, 0x42, 0x2, 0x4aa, 0x4a9, 0x3, 0x2, + 0x2, 0x2, 0x4aa, 0x4ab, 0x3, 0x2, 0x2, 0x2, 0x4ab, 0x4ad, 0x3, 0x2, + 0x2, 0x2, 0x4ac, 0x4ae, 0x5, 0x84, 0x43, 0x2, 0x4ad, 0x4ac, 0x3, 0x2, + 0x2, 0x2, 0x4ad, 0x4ae, 0x3, 0x2, 0x2, 0x2, 0x4ae, 0x4b0, 0x3, 0x2, + 0x2, 0x2, 0x4af, 0x4b1, 0x5, 0x86, 0x44, 0x2, 0x4b0, 0x4af, 0x3, 0x2, + 0x2, 0x2, 0x4b0, 0x4b1, 0x3, 0x2, 0x2, 0x2, 0x4b1, 0x6d, 0x3, 0x2, 0x2, + 0x2, 0x4b2, 0x4b3, 0x7, 0xb6, 0x2, 0x2, 0x4b3, 0x4b4, 0x5, 0xac, 0x57, + 0x2, 0x4b4, 0x6f, 0x3, 0x2, 0x2, 0x2, 0x4b5, 0x4b6, 0x7, 0xa3, 0x2, + 0x2, 0x4b6, 0x4b9, 0x7, 0xbd, 0x2, 0x2, 0x4b7, 0x4b8, 0x7, 0xb6, 0x2, + 0x2, 0x4b8, 0x4ba, 0x7, 0x9f, 0x2, 0x2, 0x4b9, 0x4b7, 0x3, 0x2, 0x2, + 0x2, 0x4b9, 0x4ba, 0x3, 0x2, 0x2, 0x2, 0x4ba, 0x71, 0x3, 0x2, 0x2, 0x2, + 0x4bb, 0x4bc, 0x7, 0x43, 0x2, 0x2, 0x4bc, 0x4bd, 0x5, 0x88, 0x45, 0x2, + 0x4bd, 0x73, 0x3, 0x2, 0x2, 0x2, 0x4be, 0x4c0, 0x9, 0x9, 0x2, 0x2, 0x4bf, + 0x4be, 0x3, 0x2, 0x2, 0x2, 0x4bf, 0x4c0, 0x3, 0x2, 0x2, 0x2, 0x4c0, + 0x4c1, 0x3, 0x2, 0x2, 0x2, 0x4c1, 0x4c2, 0x7, 0xb, 0x2, 0x2, 0x4c2, + 0x4c3, 0x7, 0x59, 0x2, 0x2, 0x4c3, 0x4c4, 0x5, 0xac, 0x57, 0x2, 0x4c4, + 0x75, 0x3, 0x2, 0x2, 0x2, 0x4c5, 0x4c6, 0x7, 0x7e, 0x2, 0x2, 0x4c6, + 0x4c7, 0x5, 0xb0, 0x59, 0x2, 0x4c7, 0x77, 0x3, 0x2, 0x2, 0x2, 0x4c8, + 0x4c9, 0x7, 0xb5, 0x2, 0x2, 0x4c9, 0x4ca, 0x5, 0xb0, 0x59, 0x2, 0x4ca, + 0x79, 0x3, 0x2, 0x2, 0x2, 0x4cb, 0x4cc, 0x7, 0x48, 0x2, 0x2, 0x4cc, + 0x4d3, 0x7, 0x14, 0x2, 0x2, 0x4cd, 0x4ce, 0x9, 0x8, 0x2, 0x2, 0x4ce, + 0x4cf, 0x7, 0xd0, 0x2, 0x2, 0x4cf, 0x4d0, 0x5, 0xac, 0x57, 0x2, 0x4d0, + 0x4d1, 0x7, 0xda, 0x2, 0x2, 0x4d1, 0x4d4, 0x3, 0x2, 0x2, 0x2, 0x4d2, + 0x4d4, 0x5, 0xac, 0x57, 0x2, 0x4d3, 0x4cd, 0x3, 0x2, 0x2, 0x2, 0x4d3, + 0x4d2, 0x3, 0x2, 0x2, 0x2, 0x4d4, 0x7b, 0x3, 0x2, 0x2, 0x2, 0x4d5, 0x4d6, + 0x7, 0x49, 0x2, 0x2, 0x4d6, 0x4d7, 0x5, 0xb0, 0x59, 0x2, 0x4d7, 0x7d, + 0x3, 0x2, 0x2, 0x2, 0x4d8, 0x4d9, 0x7, 0x79, 0x2, 0x2, 0x4d9, 0x4da, + 0x7, 0x14, 0x2, 0x2, 0x4da, 0x4db, 0x5, 0x94, 0x4b, 0x2, 0x4db, 0x7f, + 0x3, 0x2, 0x2, 0x2, 0x4dc, 0x4dd, 0x7, 0x79, 0x2, 0x2, 0x4dd, 0x4de, + 0x7, 0x14, 0x2, 0x2, 0x4de, 0x4df, 0x5, 0xac, 0x57, 0x2, 0x4df, 0x81, + 0x3, 0x2, 0x2, 0x2, 0x4e0, 0x4e1, 0x7, 0x62, 0x2, 0x2, 0x4e1, 0x4e2, + 0x5, 0x92, 0x4a, 0x2, 0x4e2, 0x4e3, 0x7, 0x14, 0x2, 0x2, 0x4e3, 0x4e4, + 0x5, 0xac, 0x57, 0x2, 0x4e4, 0x83, 0x3, 0x2, 0x2, 0x2, 0x4e5, 0x4e6, + 0x7, 0x62, 0x2, 0x2, 0x4e6, 0x4e9, 0x5, 0x92, 0x4a, 0x2, 0x4e7, 0x4e8, + 0x7, 0xb6, 0x2, 0x2, 0x4e8, 0x4ea, 0x7, 0x9f, 0x2, 0x2, 0x4e9, 0x4e7, + 0x3, 0x2, 0x2, 0x2, 0x4e9, 0x4ea, 0x3, 0x2, 0x2, 0x2, 0x4ea, 0x85, 0x3, + 0x2, 0x2, 0x2, 0x4eb, 0x4ec, 0x7, 0x91, 0x2, 0x2, 0x4ec, 0x4ed, 0x5, + 0x9a, 0x4e, 0x2, 0x4ed, 0x87, 0x3, 0x2, 0x2, 0x2, 0x4ee, 0x4ef, 0x8, + 0x45, 0x1, 0x2, 0x4ef, 0x4f1, 0x5, 0xbc, 0x5f, 0x2, 0x4f0, 0x4f2, 0x7, + 0x3d, 0x2, 0x2, 0x4f1, 0x4f0, 0x3, 0x2, 0x2, 0x2, 0x4f1, 0x4f2, 0x3, + 0x2, 0x2, 0x2, 0x4f2, 0x4f4, 0x3, 0x2, 0x2, 0x2, 0x4f3, 0x4f5, 0x5, + 0x90, 0x49, 0x2, 0x4f4, 0x4f3, 0x3, 0x2, 0x2, 0x2, 0x4f4, 0x4f5, 0x3, + 0x2, 0x2, 0x2, 0x4f5, 0x4fb, 0x3, 0x2, 0x2, 0x2, 0x4f6, 0x4f7, 0x7, + 0xd0, 0x2, 0x2, 0x4f7, 0x4f8, 0x5, 0x88, 0x45, 0x2, 0x4f8, 0x4f9, 0x7, + 0xda, 0x2, 0x2, 0x4f9, 0x4fb, 0x3, 0x2, 0x2, 0x2, 0x4fa, 0x4ee, 0x3, + 0x2, 0x2, 0x2, 0x4fa, 0x4f6, 0x3, 0x2, 0x2, 0x2, 0x4fb, 0x50d, 0x3, + 0x2, 0x2, 0x2, 0x4fc, 0x4fd, 0xc, 0x5, 0x2, 0x2, 0x4fd, 0x4fe, 0x5, + 0x8c, 0x47, 0x2, 0x4fe, 0x4ff, 0x5, 0x88, 0x45, 0x6, 0x4ff, 0x50c, 0x3, + 0x2, 0x2, 0x2, 0x500, 0x502, 0xc, 0x6, 0x2, 0x2, 0x501, 0x503, 0x9, + 0xa, 0x2, 0x2, 0x502, 0x501, 0x3, 0x2, 0x2, 0x2, 0x502, 0x503, 0x3, + 0x2, 0x2, 0x2, 0x503, 0x505, 0x3, 0x2, 0x2, 0x2, 0x504, 0x506, 0x5, + 0x8a, 0x46, 0x2, 0x505, 0x504, 0x3, 0x2, 0x2, 0x2, 0x505, 0x506, 0x3, + 0x2, 0x2, 0x2, 0x506, 0x507, 0x3, 0x2, 0x2, 0x2, 0x507, 0x508, 0x7, + 0x59, 0x2, 0x2, 0x508, 0x509, 0x5, 0x88, 0x45, 0x2, 0x509, 0x50a, 0x5, + 0x8e, 0x48, 0x2, 0x50a, 0x50c, 0x3, 0x2, 0x2, 0x2, 0x50b, 0x4fc, 0x3, + 0x2, 0x2, 0x2, 0x50b, 0x500, 0x3, 0x2, 0x2, 0x2, 0x50c, 0x50f, 0x3, + 0x2, 0x2, 0x2, 0x50d, 0x50b, 0x3, 0x2, 0x2, 0x2, 0x50d, 0x50e, 0x3, + 0x2, 0x2, 0x2, 0x50e, 0x89, 0x3, 0x2, 0x2, 0x2, 0x50f, 0x50d, 0x3, 0x2, + 0x2, 0x2, 0x510, 0x512, 0x9, 0xb, 0x2, 0x2, 0x511, 0x510, 0x3, 0x2, + 0x2, 0x2, 0x511, 0x512, 0x3, 0x2, 0x2, 0x2, 0x512, 0x513, 0x3, 0x2, + 0x2, 0x2, 0x513, 0x51a, 0x7, 0x53, 0x2, 0x2, 0x514, 0x516, 0x7, 0x53, + 0x2, 0x2, 0x515, 0x517, 0x9, 0xb, 0x2, 0x2, 0x516, 0x515, 0x3, 0x2, + 0x2, 0x2, 0x516, 0x517, 0x3, 0x2, 0x2, 0x2, 0x517, 0x51a, 0x3, 0x2, + 0x2, 0x2, 0x518, 0x51a, 0x9, 0xb, 0x2, 0x2, 0x519, 0x511, 0x3, 0x2, + 0x2, 0x2, 0x519, 0x514, 0x3, 0x2, 0x2, 0x2, 0x519, 0x518, 0x3, 0x2, + 0x2, 0x2, 0x51a, 0x53c, 0x3, 0x2, 0x2, 0x2, 0x51b, 0x51d, 0x9, 0xc, + 0x2, 0x2, 0x51c, 0x51b, 0x3, 0x2, 0x2, 0x2, 0x51c, 0x51d, 0x3, 0x2, + 0x2, 0x2, 0x51d, 0x51e, 0x3, 0x2, 0x2, 0x2, 0x51e, 0x520, 0x9, 0xd, + 0x2, 0x2, 0x51f, 0x521, 0x7, 0x7a, 0x2, 0x2, 0x520, 0x51f, 0x3, 0x2, + 0x2, 0x2, 0x520, 0x521, 0x3, 0x2, 0x2, 0x2, 0x521, 0x52a, 0x3, 0x2, + 0x2, 0x2, 0x522, 0x524, 0x9, 0xd, 0x2, 0x2, 0x523, 0x525, 0x7, 0x7a, + 0x2, 0x2, 0x524, 0x523, 0x3, 0x2, 0x2, 0x2, 0x524, 0x525, 0x3, 0x2, + 0x2, 0x2, 0x525, 0x527, 0x3, 0x2, 0x2, 0x2, 0x526, 0x528, 0x9, 0xc, + 0x2, 0x2, 0x527, 0x526, 0x3, 0x2, 0x2, 0x2, 0x527, 0x528, 0x3, 0x2, + 0x2, 0x2, 0x528, 0x52a, 0x3, 0x2, 0x2, 0x2, 0x529, 0x51c, 0x3, 0x2, + 0x2, 0x2, 0x529, 0x522, 0x3, 0x2, 0x2, 0x2, 0x52a, 0x53c, 0x3, 0x2, + 0x2, 0x2, 0x52b, 0x52d, 0x9, 0xe, 0x2, 0x2, 0x52c, 0x52b, 0x3, 0x2, + 0x2, 0x2, 0x52c, 0x52d, 0x3, 0x2, 0x2, 0x2, 0x52d, 0x52e, 0x3, 0x2, + 0x2, 0x2, 0x52e, 0x530, 0x7, 0x44, 0x2, 0x2, 0x52f, 0x531, 0x7, 0x7a, + 0x2, 0x2, 0x530, 0x52f, 0x3, 0x2, 0x2, 0x2, 0x530, 0x531, 0x3, 0x2, + 0x2, 0x2, 0x531, 0x53a, 0x3, 0x2, 0x2, 0x2, 0x532, 0x534, 0x7, 0x44, + 0x2, 0x2, 0x533, 0x535, 0x7, 0x7a, 0x2, 0x2, 0x534, 0x533, 0x3, 0x2, + 0x2, 0x2, 0x534, 0x535, 0x3, 0x2, 0x2, 0x2, 0x535, 0x537, 0x3, 0x2, + 0x2, 0x2, 0x536, 0x538, 0x9, 0xe, 0x2, 0x2, 0x537, 0x536, 0x3, 0x2, + 0x2, 0x2, 0x537, 0x538, 0x3, 0x2, 0x2, 0x2, 0x538, 0x53a, 0x3, 0x2, + 0x2, 0x2, 0x539, 0x52c, 0x3, 0x2, 0x2, 0x2, 0x539, 0x532, 0x3, 0x2, + 0x2, 0x2, 0x53a, 0x53c, 0x3, 0x2, 0x2, 0x2, 0x53b, 0x519, 0x3, 0x2, + 0x2, 0x2, 0x53b, 0x529, 0x3, 0x2, 0x2, 0x2, 0x53b, 0x539, 0x3, 0x2, + 0x2, 0x2, 0x53c, 0x8b, 0x3, 0x2, 0x2, 0x2, 0x53d, 0x53f, 0x9, 0xa, 0x2, + 0x2, 0x53e, 0x53d, 0x3, 0x2, 0x2, 0x2, 0x53e, 0x53f, 0x3, 0x2, 0x2, + 0x2, 0x53f, 0x540, 0x3, 0x2, 0x2, 0x2, 0x540, 0x541, 0x7, 0x20, 0x2, + 0x2, 0x541, 0x544, 0x7, 0x59, 0x2, 0x2, 0x542, 0x544, 0x7, 0xc5, 0x2, + 0x2, 0x543, 0x53e, 0x3, 0x2, 0x2, 0x2, 0x543, 0x542, 0x3, 0x2, 0x2, + 0x2, 0x544, 0x8d, 0x3, 0x2, 0x2, 0x2, 0x545, 0x546, 0x7, 0x76, 0x2, + 0x2, 0x546, 0x54f, 0x5, 0xac, 0x57, 0x2, 0x547, 0x548, 0x7, 0xad, 0x2, + 0x2, 0x548, 0x549, 0x7, 0xd0, 0x2, 0x2, 0x549, 0x54a, 0x5, 0xac, 0x57, + 0x2, 0x54a, 0x54b, 0x7, 0xda, 0x2, 0x2, 0x54b, 0x54f, 0x3, 0x2, 0x2, + 0x2, 0x54c, 0x54d, 0x7, 0xad, 0x2, 0x2, 0x54d, 0x54f, 0x5, 0xac, 0x57, + 0x2, 0x54e, 0x545, 0x3, 0x2, 0x2, 0x2, 0x54e, 0x547, 0x3, 0x2, 0x2, + 0x2, 0x54e, 0x54c, 0x3, 0x2, 0x2, 0x2, 0x54f, 0x8f, 0x3, 0x2, 0x2, 0x2, + 0x550, 0x551, 0x7, 0x8b, 0x2, 0x2, 0x551, 0x554, 0x5, 0x98, 0x4d, 0x2, + 0x552, 0x553, 0x7, 0x75, 0x2, 0x2, 0x553, 0x555, 0x5, 0x98, 0x4d, 0x2, + 0x554, 0x552, 0x3, 0x2, 0x2, 0x2, 0x554, 0x555, 0x3, 0x2, 0x2, 0x2, + 0x555, 0x91, 0x3, 0x2, 0x2, 0x2, 0x556, 0x559, 0x5, 0xb0, 0x59, 0x2, + 0x557, 0x558, 0x9, 0xf, 0x2, 0x2, 0x558, 0x55a, 0x5, 0xb0, 0x59, 0x2, + 0x559, 0x557, 0x3, 0x2, 0x2, 0x2, 0x559, 0x55a, 0x3, 0x2, 0x2, 0x2, + 0x55a, 0x93, 0x3, 0x2, 0x2, 0x2, 0x55b, 0x560, 0x5, 0x96, 0x4c, 0x2, + 0x55c, 0x55d, 0x7, 0xc5, 0x2, 0x2, 0x55d, 0x55f, 0x5, 0x96, 0x4c, 0x2, + 0x55e, 0x55c, 0x3, 0x2, 0x2, 0x2, 0x55f, 0x562, 0x3, 0x2, 0x2, 0x2, + 0x560, 0x55e, 0x3, 0x2, 0x2, 0x2, 0x560, 0x561, 0x3, 0x2, 0x2, 0x2, + 0x561, 0x95, 0x3, 0x2, 0x2, 0x2, 0x562, 0x560, 0x3, 0x2, 0x2, 0x2, 0x563, + 0x565, 0x5, 0xb0, 0x59, 0x2, 0x564, 0x566, 0x9, 0x10, 0x2, 0x2, 0x565, + 0x564, 0x3, 0x2, 0x2, 0x2, 0x565, 0x566, 0x3, 0x2, 0x2, 0x2, 0x566, + 0x569, 0x3, 0x2, 0x2, 0x2, 0x567, 0x568, 0x7, 0x74, 0x2, 0x2, 0x568, + 0x56a, 0x9, 0x11, 0x2, 0x2, 0x569, 0x567, 0x3, 0x2, 0x2, 0x2, 0x569, + 0x56a, 0x3, 0x2, 0x2, 0x2, 0x56a, 0x56d, 0x3, 0x2, 0x2, 0x2, 0x56b, + 0x56c, 0x7, 0x1b, 0x2, 0x2, 0x56c, 0x56e, 0x7, 0xbf, 0x2, 0x2, 0x56d, + 0x56b, 0x3, 0x2, 0x2, 0x2, 0x56d, 0x56e, 0x3, 0x2, 0x2, 0x2, 0x56e, + 0x97, 0x3, 0x2, 0x2, 0x2, 0x56f, 0x572, 0x5, 0xca, 0x66, 0x2, 0x570, + 0x571, 0x7, 0xdc, 0x2, 0x2, 0x571, 0x573, 0x5, 0xca, 0x66, 0x2, 0x572, + 0x570, 0x3, 0x2, 0x2, 0x2, 0x572, 0x573, 0x3, 0x2, 0x2, 0x2, 0x573, + 0x99, 0x3, 0x2, 0x2, 0x2, 0x574, 0x579, 0x5, 0x9c, 0x4f, 0x2, 0x575, + 0x576, 0x7, 0xc5, 0x2, 0x2, 0x576, 0x578, 0x5, 0x9c, 0x4f, 0x2, 0x577, + 0x575, 0x3, 0x2, 0x2, 0x2, 0x578, 0x57b, 0x3, 0x2, 0x2, 0x2, 0x579, + 0x577, 0x3, 0x2, 0x2, 0x2, 0x579, 0x57a, 0x3, 0x2, 0x2, 0x2, 0x57a, + 0x9b, 0x3, 0x2, 0x2, 0x2, 0x57b, 0x579, 0x3, 0x2, 0x2, 0x2, 0x57c, 0x57d, + 0x5, 0xd6, 0x6c, 0x2, 0x57d, 0x57e, 0x7, 0xca, 0x2, 0x2, 0x57e, 0x57f, + 0x5, 0xcc, 0x67, 0x2, 0x57f, 0x9d, 0x3, 0x2, 0x2, 0x2, 0x580, 0x581, + 0x7, 0x90, 0x2, 0x2, 0x581, 0x582, 0x5, 0x9a, 0x4e, 0x2, 0x582, 0x9f, + 0x3, 0x2, 0x2, 0x2, 0x583, 0x584, 0x7, 0x92, 0x2, 0x2, 0x584, 0x585, + 0x7, 0x1f, 0x2, 0x2, 0x585, 0x586, 0x7, 0x22, 0x2, 0x2, 0x586, 0x5ae, + 0x5, 0xc6, 0x64, 0x2, 0x587, 0x588, 0x7, 0x92, 0x2, 0x2, 0x588, 0x589, + 0x7, 0x1f, 0x2, 0x2, 0x589, 0x58a, 0x7, 0x2f, 0x2, 0x2, 0x58a, 0x5ae, + 0x5, 0xc0, 0x61, 0x2, 0x58b, 0x58c, 0x7, 0x92, 0x2, 0x2, 0x58c, 0x58e, + 0x7, 0x1f, 0x2, 0x2, 0x58d, 0x58f, 0x7, 0x9c, 0x2, 0x2, 0x58e, 0x58d, + 0x3, 0x2, 0x2, 0x2, 0x58e, 0x58f, 0x3, 0x2, 0x2, 0x2, 0x58f, 0x591, + 0x3, 0x2, 0x2, 0x2, 0x590, 0x592, 0x7, 0x9a, 0x2, 0x2, 0x591, 0x590, + 0x3, 0x2, 0x2, 0x2, 0x591, 0x592, 0x3, 0x2, 0x2, 0x2, 0x592, 0x593, + 0x3, 0x2, 0x2, 0x2, 0x593, 0x5ae, 0x5, 0xc0, 0x61, 0x2, 0x594, 0x595, + 0x7, 0x92, 0x2, 0x2, 0x595, 0x5ae, 0x7, 0x23, 0x2, 0x2, 0x596, 0x597, + 0x7, 0x92, 0x2, 0x2, 0x597, 0x59a, 0x7, 0x2e, 0x2, 0x2, 0x598, 0x599, + 0x7, 0x43, 0x2, 0x2, 0x599, 0x59b, 0x5, 0xc6, 0x64, 0x2, 0x59a, 0x598, + 0x3, 0x2, 0x2, 0x2, 0x59a, 0x59b, 0x3, 0x2, 0x2, 0x2, 0x59b, 0x5ae, + 0x3, 0x2, 0x2, 0x2, 0x59c, 0x59e, 0x7, 0x92, 0x2, 0x2, 0x59d, 0x59f, + 0x7, 0x9c, 0x2, 0x2, 0x59e, 0x59d, 0x3, 0x2, 0x2, 0x2, 0x59e, 0x59f, + 0x3, 0x2, 0x2, 0x2, 0x59f, 0x5a0, 0x3, 0x2, 0x2, 0x2, 0x5a0, 0x5a3, + 0x7, 0x9b, 0x2, 0x2, 0x5a1, 0x5a2, 0x9, 0x12, 0x2, 0x2, 0x5a2, 0x5a4, + 0x5, 0xc6, 0x64, 0x2, 0x5a3, 0x5a1, 0x3, 0x2, 0x2, 0x2, 0x5a3, 0x5a4, + 0x3, 0x2, 0x2, 0x2, 0x5a4, 0x5a8, 0x3, 0x2, 0x2, 0x2, 0x5a5, 0x5a6, + 0x7, 0x61, 0x2, 0x2, 0x5a6, 0x5a9, 0x7, 0xbf, 0x2, 0x2, 0x5a7, 0x5a9, + 0x5, 0x78, 0x3d, 0x2, 0x5a8, 0x5a5, 0x3, 0x2, 0x2, 0x2, 0x5a8, 0x5a7, + 0x3, 0x2, 0x2, 0x2, 0x5a8, 0x5a9, 0x3, 0x2, 0x2, 0x2, 0x5a9, 0x5ab, + 0x3, 0x2, 0x2, 0x2, 0x5aa, 0x5ac, 0x5, 0x84, 0x43, 0x2, 0x5ab, 0x5aa, + 0x3, 0x2, 0x2, 0x2, 0x5ab, 0x5ac, 0x3, 0x2, 0x2, 0x2, 0x5ac, 0x5ae, + 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x583, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x587, + 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x58b, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x594, + 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x596, 0x3, 0x2, 0x2, 0x2, 0x5ad, 0x59c, + 0x3, 0x2, 0x2, 0x2, 0x5ae, 0xa1, 0x3, 0x2, 0x2, 0x2, 0x5af, 0x5b0, 0x7, + 0x99, 0x2, 0x2, 0x5b0, 0x5b1, 0x7, 0x3f, 0x2, 0x2, 0x5b1, 0x5b2, 0x7, + 0x32, 0x2, 0x2, 0x5b2, 0x5d2, 0x5, 0xc0, 0x61, 0x2, 0x5b3, 0x5b4, 0x7, + 0x99, 0x2, 0x2, 0x5b4, 0x5b5, 0x7, 0x3f, 0x2, 0x2, 0x5b5, 0x5d2, 0x7, + 0x65, 0x2, 0x2, 0x5b6, 0x5b7, 0x7, 0x99, 0x2, 0x2, 0x5b7, 0x5b8, 0x7, + 0x83, 0x2, 0x2, 0x5b8, 0x5d2, 0x7, 0x2e, 0x2, 0x2, 0x5b9, 0x5ba, 0x7, + 0x99, 0x2, 0x2, 0x5ba, 0x5bb, 0x7, 0x83, 0x2, 0x2, 0x5bb, 0x5bc, 0x7, + 0x2f, 0x2, 0x2, 0x5bc, 0x5d2, 0x5, 0xc0, 0x61, 0x2, 0x5bd, 0x5be, 0x7, + 0x99, 0x2, 0x2, 0x5be, 0x5c6, 0x9, 0x13, 0x2, 0x2, 0x5bf, 0x5c0, 0x7, + 0x32, 0x2, 0x2, 0x5c0, 0x5c7, 0x7, 0x8f, 0x2, 0x2, 0x5c1, 0x5c7, 0x7, + 0x3c, 0x2, 0x2, 0x5c2, 0x5c4, 0x7, 0xa8, 0x2, 0x2, 0x5c3, 0x5c2, 0x3, + 0x2, 0x2, 0x2, 0x5c3, 0x5c4, 0x3, 0x2, 0x2, 0x2, 0x5c4, 0x5c5, 0x3, + 0x2, 0x2, 0x2, 0x5c5, 0x5c7, 0x7, 0x69, 0x2, 0x2, 0x5c6, 0x5bf, 0x3, + 0x2, 0x2, 0x2, 0x5c6, 0x5c1, 0x3, 0x2, 0x2, 0x2, 0x5c6, 0x5c3, 0x3, + 0x2, 0x2, 0x2, 0x5c7, 0x5c8, 0x3, 0x2, 0x2, 0x2, 0x5c8, 0x5d2, 0x5, + 0xc0, 0x61, 0x2, 0x5c9, 0x5ca, 0x7, 0x99, 0x2, 0x2, 0x5ca, 0x5cb, 0x9, + 0x13, 0x2, 0x2, 0x5cb, 0x5cc, 0x7, 0x88, 0x2, 0x2, 0x5cc, 0x5d2, 0x7, + 0x8f, 0x2, 0x2, 0x5cd, 0x5ce, 0x7, 0x99, 0x2, 0x2, 0x5ce, 0x5cf, 0x7, + 0x97, 0x2, 0x2, 0x5cf, 0x5d0, 0x7, 0x87, 0x2, 0x2, 0x5d0, 0x5d2, 0x5, + 0xc0, 0x61, 0x2, 0x5d1, 0x5af, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5b3, 0x3, + 0x2, 0x2, 0x2, 0x5d1, 0x5b6, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5b9, 0x3, + 0x2, 0x2, 0x2, 0x5d1, 0x5bd, 0x3, 0x2, 0x2, 0x2, 0x5d1, 0x5c9, 0x3, + 0x2, 0x2, 0x2, 0x5d1, 0x5cd, 0x3, 0x2, 0x2, 0x2, 0x5d2, 0xa3, 0x3, 0x2, + 0x2, 0x2, 0x5d3, 0x5d5, 0x7, 0xa7, 0x2, 0x2, 0x5d4, 0x5d6, 0x7, 0x9c, + 0x2, 0x2, 0x5d5, 0x5d4, 0x3, 0x2, 0x2, 0x2, 0x5d5, 0x5d6, 0x3, 0x2, + 0x2, 0x2, 0x5d6, 0x5d8, 0x3, 0x2, 0x2, 0x2, 0x5d7, 0x5d9, 0x7, 0x9a, + 0x2, 0x2, 0x5d8, 0x5d7, 0x3, 0x2, 0x2, 0x2, 0x5d8, 0x5d9, 0x3, 0x2, + 0x2, 0x2, 0x5d9, 0x5dc, 0x3, 0x2, 0x2, 0x2, 0x5da, 0x5db, 0x7, 0x4d, + 0x2, 0x2, 0x5db, 0x5dd, 0x7, 0x38, 0x2, 0x2, 0x5dc, 0x5da, 0x3, 0x2, + 0x2, 0x2, 0x5dc, 0x5dd, 0x3, 0x2, 0x2, 0x2, 0x5dd, 0x5de, 0x3, 0x2, + 0x2, 0x2, 0x5de, 0x5e0, 0x5, 0xc0, 0x61, 0x2, 0x5df, 0x5e1, 0x5, 0x2c, + 0x17, 0x2, 0x5e0, 0x5df, 0x3, 0x2, 0x2, 0x2, 0x5e0, 0x5e1, 0x3, 0x2, + 0x2, 0x2, 0x5e1, 0xa5, 0x3, 0x2, 0x2, 0x2, 0x5e2, 0x5e3, 0x7, 0xac, + 0x2, 0x2, 0x5e3, 0x5e4, 0x5, 0xc6, 0x64, 0x2, 0x5e4, 0xa7, 0x3, 0x2, + 0x2, 0x2, 0x5e5, 0x5e6, 0x7, 0xb2, 0x2, 0x2, 0x5e6, 0x5e8, 0x5, 0xc0, + 0x61, 0x2, 0x5e7, 0x5e9, 0x7, 0x37, 0x2, 0x2, 0x5e8, 0x5e7, 0x3, 0x2, + 0x2, 0x2, 0x5e8, 0x5e9, 0x3, 0x2, 0x2, 0x2, 0x5e9, 0x5ec, 0x3, 0x2, + 0x2, 0x2, 0x5ea, 0x5eb, 0x7, 0x62, 0x2, 0x2, 0x5eb, 0x5ed, 0x7, 0xbd, + 0x2, 0x2, 0x5ec, 0x5ea, 0x3, 0x2, 0x2, 0x2, 0x5ec, 0x5ed, 0x3, 0x2, + 0x2, 0x2, 0x5ed, 0xa9, 0x3, 0x2, 0x2, 0x2, 0x5ee, 0x61e, 0x5, 0xd6, + 0x6c, 0x2, 0x5ef, 0x5f0, 0x5, 0xd6, 0x6c, 0x2, 0x5f0, 0x5f1, 0x7, 0xd0, + 0x2, 0x2, 0x5f1, 0x5f2, 0x5, 0xd6, 0x6c, 0x2, 0x5f2, 0x5f9, 0x5, 0xaa, + 0x56, 0x2, 0x5f3, 0x5f4, 0x7, 0xc5, 0x2, 0x2, 0x5f4, 0x5f5, 0x5, 0xd6, + 0x6c, 0x2, 0x5f5, 0x5f6, 0x5, 0xaa, 0x56, 0x2, 0x5f6, 0x5f8, 0x3, 0x2, + 0x2, 0x2, 0x5f7, 0x5f3, 0x3, 0x2, 0x2, 0x2, 0x5f8, 0x5fb, 0x3, 0x2, + 0x2, 0x2, 0x5f9, 0x5f7, 0x3, 0x2, 0x2, 0x2, 0x5f9, 0x5fa, 0x3, 0x2, + 0x2, 0x2, 0x5fa, 0x5fc, 0x3, 0x2, 0x2, 0x2, 0x5fb, 0x5f9, 0x3, 0x2, + 0x2, 0x2, 0x5fc, 0x5fd, 0x7, 0xda, 0x2, 0x2, 0x5fd, 0x61e, 0x3, 0x2, + 0x2, 0x2, 0x5fe, 0x5ff, 0x5, 0xd6, 0x6c, 0x2, 0x5ff, 0x600, 0x7, 0xd0, + 0x2, 0x2, 0x600, 0x605, 0x5, 0xda, 0x6e, 0x2, 0x601, 0x602, 0x7, 0xc5, + 0x2, 0x2, 0x602, 0x604, 0x5, 0xda, 0x6e, 0x2, 0x603, 0x601, 0x3, 0x2, + 0x2, 0x2, 0x604, 0x607, 0x3, 0x2, 0x2, 0x2, 0x605, 0x603, 0x3, 0x2, + 0x2, 0x2, 0x605, 0x606, 0x3, 0x2, 0x2, 0x2, 0x606, 0x608, 0x3, 0x2, + 0x2, 0x2, 0x607, 0x605, 0x3, 0x2, 0x2, 0x2, 0x608, 0x609, 0x7, 0xda, + 0x2, 0x2, 0x609, 0x61e, 0x3, 0x2, 0x2, 0x2, 0x60a, 0x60b, 0x5, 0xd6, + 0x6c, 0x2, 0x60b, 0x60c, 0x7, 0xd0, 0x2, 0x2, 0x60c, 0x611, 0x5, 0xaa, + 0x56, 0x2, 0x60d, 0x60e, 0x7, 0xc5, 0x2, 0x2, 0x60e, 0x610, 0x5, 0xaa, + 0x56, 0x2, 0x60f, 0x60d, 0x3, 0x2, 0x2, 0x2, 0x610, 0x613, 0x3, 0x2, + 0x2, 0x2, 0x611, 0x60f, 0x3, 0x2, 0x2, 0x2, 0x611, 0x612, 0x3, 0x2, + 0x2, 0x2, 0x612, 0x614, 0x3, 0x2, 0x2, 0x2, 0x613, 0x611, 0x3, 0x2, + 0x2, 0x2, 0x614, 0x615, 0x7, 0xda, 0x2, 0x2, 0x615, 0x61e, 0x3, 0x2, + 0x2, 0x2, 0x616, 0x617, 0x5, 0xd6, 0x6c, 0x2, 0x617, 0x619, 0x7, 0xd0, + 0x2, 0x2, 0x618, 0x61a, 0x5, 0xac, 0x57, 0x2, 0x619, 0x618, 0x3, 0x2, + 0x2, 0x2, 0x619, 0x61a, 0x3, 0x2, 0x2, 0x2, 0x61a, 0x61b, 0x3, 0x2, + 0x2, 0x2, 0x61b, 0x61c, 0x7, 0xda, 0x2, 0x2, 0x61c, 0x61e, 0x3, 0x2, + 0x2, 0x2, 0x61d, 0x5ee, 0x3, 0x2, 0x2, 0x2, 0x61d, 0x5ef, 0x3, 0x2, + 0x2, 0x2, 0x61d, 0x5fe, 0x3, 0x2, 0x2, 0x2, 0x61d, 0x60a, 0x3, 0x2, + 0x2, 0x2, 0x61d, 0x616, 0x3, 0x2, 0x2, 0x2, 0x61e, 0xab, 0x3, 0x2, 0x2, + 0x2, 0x61f, 0x624, 0x5, 0xae, 0x58, 0x2, 0x620, 0x621, 0x7, 0xc5, 0x2, + 0x2, 0x621, 0x623, 0x5, 0xae, 0x58, 0x2, 0x622, 0x620, 0x3, 0x2, 0x2, + 0x2, 0x623, 0x626, 0x3, 0x2, 0x2, 0x2, 0x624, 0x622, 0x3, 0x2, 0x2, + 0x2, 0x624, 0x625, 0x3, 0x2, 0x2, 0x2, 0x625, 0xad, 0x3, 0x2, 0x2, 0x2, + 0x626, 0x624, 0x3, 0x2, 0x2, 0x2, 0x627, 0x628, 0x5, 0xc0, 0x61, 0x2, + 0x628, 0x629, 0x7, 0xc8, 0x2, 0x2, 0x629, 0x62b, 0x3, 0x2, 0x2, 0x2, + 0x62a, 0x627, 0x3, 0x2, 0x2, 0x2, 0x62a, 0x62b, 0x3, 0x2, 0x2, 0x2, + 0x62b, 0x62c, 0x3, 0x2, 0x2, 0x2, 0x62c, 0x633, 0x7, 0xc1, 0x2, 0x2, + 0x62d, 0x62e, 0x7, 0xd0, 0x2, 0x2, 0x62e, 0x62f, 0x5, 0x68, 0x35, 0x2, + 0x62f, 0x630, 0x7, 0xda, 0x2, 0x2, 0x630, 0x633, 0x3, 0x2, 0x2, 0x2, + 0x631, 0x633, 0x5, 0xb0, 0x59, 0x2, 0x632, 0x62a, 0x3, 0x2, 0x2, 0x2, + 0x632, 0x62d, 0x3, 0x2, 0x2, 0x2, 0x632, 0x631, 0x3, 0x2, 0x2, 0x2, + 0x633, 0xaf, 0x3, 0x2, 0x2, 0x2, 0x634, 0x635, 0x8, 0x59, 0x1, 0x2, + 0x635, 0x637, 0x7, 0x15, 0x2, 0x2, 0x636, 0x638, 0x5, 0xb0, 0x59, 0x2, + 0x637, 0x636, 0x3, 0x2, 0x2, 0x2, 0x637, 0x638, 0x3, 0x2, 0x2, 0x2, + 0x638, 0x63e, 0x3, 0x2, 0x2, 0x2, 0x639, 0x63a, 0x7, 0xb4, 0x2, 0x2, + 0x63a, 0x63b, 0x5, 0xb0, 0x59, 0x2, 0x63b, 0x63c, 0x7, 0x9e, 0x2, 0x2, + 0x63c, 0x63d, 0x5, 0xb0, 0x59, 0x2, 0x63d, 0x63f, 0x3, 0x2, 0x2, 0x2, + 0x63e, 0x639, 0x3, 0x2, 0x2, 0x2, 0x63f, 0x640, 0x3, 0x2, 0x2, 0x2, + 0x640, 0x63e, 0x3, 0x2, 0x2, 0x2, 0x640, 0x641, 0x3, 0x2, 0x2, 0x2, + 0x641, 0x644, 0x3, 0x2, 0x2, 0x2, 0x642, 0x643, 0x7, 0x34, 0x2, 0x2, + 0x643, 0x645, 0x5, 0xb0, 0x59, 0x2, 0x644, 0x642, 0x3, 0x2, 0x2, 0x2, + 0x644, 0x645, 0x3, 0x2, 0x2, 0x2, 0x645, 0x646, 0x3, 0x2, 0x2, 0x2, + 0x646, 0x647, 0x7, 0x35, 0x2, 0x2, 0x647, 0x6a0, 0x3, 0x2, 0x2, 0x2, + 0x648, 0x649, 0x7, 0x16, 0x2, 0x2, 0x649, 0x64a, 0x7, 0xd0, 0x2, 0x2, + 0x64a, 0x64b, 0x5, 0xb0, 0x59, 0x2, 0x64b, 0x64c, 0x7, 0xc, 0x2, 0x2, + 0x64c, 0x64d, 0x5, 0xaa, 0x56, 0x2, 0x64d, 0x64e, 0x7, 0xda, 0x2, 0x2, + 0x64e, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x64f, 0x650, 0x7, 0x24, 0x2, 0x2, + 0x650, 0x6a0, 0x7, 0xbf, 0x2, 0x2, 0x651, 0x652, 0x7, 0x3b, 0x2, 0x2, + 0x652, 0x653, 0x7, 0xd0, 0x2, 0x2, 0x653, 0x654, 0x5, 0xce, 0x68, 0x2, + 0x654, 0x655, 0x7, 0x43, 0x2, 0x2, 0x655, 0x656, 0x5, 0xb0, 0x59, 0x2, + 0x656, 0x657, 0x7, 0xda, 0x2, 0x2, 0x657, 0x6a0, 0x3, 0x2, 0x2, 0x2, + 0x658, 0x659, 0x7, 0x55, 0x2, 0x2, 0x659, 0x65a, 0x5, 0xb0, 0x59, 0x2, + 0x65a, 0x65b, 0x5, 0xce, 0x68, 0x2, 0x65b, 0x6a0, 0x3, 0x2, 0x2, 0x2, + 0x65c, 0x65d, 0x7, 0x96, 0x2, 0x2, 0x65d, 0x65e, 0x7, 0xd0, 0x2, 0x2, + 0x65e, 0x65f, 0x5, 0xb0, 0x59, 0x2, 0x65f, 0x660, 0x7, 0x43, 0x2, 0x2, + 0x660, 0x663, 0x5, 0xb0, 0x59, 0x2, 0x661, 0x662, 0x7, 0x40, 0x2, 0x2, + 0x662, 0x664, 0x5, 0xb0, 0x59, 0x2, 0x663, 0x661, 0x3, 0x2, 0x2, 0x2, + 0x663, 0x664, 0x3, 0x2, 0x2, 0x2, 0x664, 0x665, 0x3, 0x2, 0x2, 0x2, + 0x665, 0x666, 0x7, 0xda, 0x2, 0x2, 0x666, 0x6a0, 0x3, 0x2, 0x2, 0x2, + 0x667, 0x668, 0x7, 0xa1, 0x2, 0x2, 0x668, 0x6a0, 0x7, 0xbf, 0x2, 0x2, + 0x669, 0x66a, 0x7, 0xa6, 0x2, 0x2, 0x66a, 0x66b, 0x7, 0xd0, 0x2, 0x2, + 0x66b, 0x66c, 0x9, 0x14, 0x2, 0x2, 0x66c, 0x66d, 0x7, 0xbf, 0x2, 0x2, + 0x66d, 0x66e, 0x7, 0x43, 0x2, 0x2, 0x66e, 0x66f, 0x5, 0xb0, 0x59, 0x2, + 0x66f, 0x670, 0x7, 0xda, 0x2, 0x2, 0x670, 0x6a0, 0x3, 0x2, 0x2, 0x2, + 0x671, 0x677, 0x5, 0xd6, 0x6c, 0x2, 0x672, 0x674, 0x7, 0xd0, 0x2, 0x2, + 0x673, 0x675, 0x5, 0xac, 0x57, 0x2, 0x674, 0x673, 0x3, 0x2, 0x2, 0x2, + 0x674, 0x675, 0x3, 0x2, 0x2, 0x2, 0x675, 0x676, 0x3, 0x2, 0x2, 0x2, + 0x676, 0x678, 0x7, 0xda, 0x2, 0x2, 0x677, 0x672, 0x3, 0x2, 0x2, 0x2, + 0x677, 0x678, 0x3, 0x2, 0x2, 0x2, 0x678, 0x679, 0x3, 0x2, 0x2, 0x2, + 0x679, 0x67b, 0x7, 0xd0, 0x2, 0x2, 0x67a, 0x67c, 0x7, 0x31, 0x2, 0x2, + 0x67b, 0x67a, 0x3, 0x2, 0x2, 0x2, 0x67b, 0x67c, 0x3, 0x2, 0x2, 0x2, + 0x67c, 0x67e, 0x3, 0x2, 0x2, 0x2, 0x67d, 0x67f, 0x5, 0xb2, 0x5a, 0x2, + 0x67e, 0x67d, 0x3, 0x2, 0x2, 0x2, 0x67e, 0x67f, 0x3, 0x2, 0x2, 0x2, + 0x67f, 0x680, 0x3, 0x2, 0x2, 0x2, 0x680, 0x681, 0x7, 0xda, 0x2, 0x2, + 0x681, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x682, 0x6a0, 0x5, 0xcc, 0x67, 0x2, + 0x683, 0x684, 0x7, 0xc7, 0x2, 0x2, 0x684, 0x6a0, 0x5, 0xb0, 0x59, 0x13, + 0x685, 0x686, 0x7, 0x72, 0x2, 0x2, 0x686, 0x6a0, 0x5, 0xb0, 0x59, 0xe, + 0x687, 0x688, 0x5, 0xc0, 0x61, 0x2, 0x688, 0x689, 0x7, 0xc8, 0x2, 0x2, + 0x689, 0x68b, 0x3, 0x2, 0x2, 0x2, 0x68a, 0x687, 0x3, 0x2, 0x2, 0x2, + 0x68a, 0x68b, 0x3, 0x2, 0x2, 0x2, 0x68b, 0x68c, 0x3, 0x2, 0x2, 0x2, + 0x68c, 0x6a0, 0x7, 0xc1, 0x2, 0x2, 0x68d, 0x68e, 0x7, 0xd0, 0x2, 0x2, + 0x68e, 0x68f, 0x5, 0x68, 0x35, 0x2, 0x68f, 0x690, 0x7, 0xda, 0x2, 0x2, + 0x690, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x691, 0x692, 0x7, 0xd0, 0x2, 0x2, + 0x692, 0x693, 0x5, 0xb0, 0x59, 0x2, 0x693, 0x694, 0x7, 0xda, 0x2, 0x2, + 0x694, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x695, 0x696, 0x7, 0xd0, 0x2, 0x2, + 0x696, 0x697, 0x5, 0xac, 0x57, 0x2, 0x697, 0x698, 0x7, 0xda, 0x2, 0x2, + 0x698, 0x6a0, 0x3, 0x2, 0x2, 0x2, 0x699, 0x69b, 0x7, 0xce, 0x2, 0x2, + 0x69a, 0x69c, 0x5, 0xac, 0x57, 0x2, 0x69b, 0x69a, 0x3, 0x2, 0x2, 0x2, + 0x69b, 0x69c, 0x3, 0x2, 0x2, 0x2, 0x69c, 0x69d, 0x3, 0x2, 0x2, 0x2, + 0x69d, 0x6a0, 0x7, 0xd9, 0x2, 0x2, 0x69e, 0x6a0, 0x5, 0xb8, 0x5d, 0x2, + 0x69f, 0x634, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x648, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x64f, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x651, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x658, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x65c, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x667, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x669, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x671, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x682, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x683, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x685, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x68a, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x68d, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x691, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x695, 0x3, 0x2, 0x2, 0x2, + 0x69f, 0x699, 0x3, 0x2, 0x2, 0x2, 0x69f, 0x69e, 0x3, 0x2, 0x2, 0x2, + 0x6a0, 0x6e8, 0x3, 0x2, 0x2, 0x2, 0x6a1, 0x6a2, 0xc, 0x12, 0x2, 0x2, + 0x6a2, 0x6a3, 0x9, 0x15, 0x2, 0x2, 0x6a3, 0x6e7, 0x5, 0xb0, 0x59, 0x13, + 0x6a4, 0x6a5, 0xc, 0x11, 0x2, 0x2, 0x6a5, 0x6a6, 0x9, 0x16, 0x2, 0x2, + 0x6a6, 0x6e7, 0x5, 0xb0, 0x59, 0x12, 0x6a7, 0x6ba, 0xc, 0x10, 0x2, 0x2, + 0x6a8, 0x6bb, 0x7, 0xc9, 0x2, 0x2, 0x6a9, 0x6bb, 0x7, 0xca, 0x2, 0x2, + 0x6aa, 0x6bb, 0x7, 0xd2, 0x2, 0x2, 0x6ab, 0x6bb, 0x7, 0xcf, 0x2, 0x2, + 0x6ac, 0x6bb, 0x7, 0xcb, 0x2, 0x2, 0x6ad, 0x6bb, 0x7, 0xd1, 0x2, 0x2, + 0x6ae, 0x6bb, 0x7, 0xcc, 0x2, 0x2, 0x6af, 0x6b1, 0x7, 0x46, 0x2, 0x2, + 0x6b0, 0x6af, 0x3, 0x2, 0x2, 0x2, 0x6b0, 0x6b1, 0x3, 0x2, 0x2, 0x2, + 0x6b1, 0x6b3, 0x3, 0x2, 0x2, 0x2, 0x6b2, 0x6b4, 0x7, 0x72, 0x2, 0x2, + 0x6b3, 0x6b2, 0x3, 0x2, 0x2, 0x2, 0x6b3, 0x6b4, 0x3, 0x2, 0x2, 0x2, + 0x6b4, 0x6b5, 0x3, 0x2, 0x2, 0x2, 0x6b5, 0x6bb, 0x7, 0x4f, 0x2, 0x2, + 0x6b6, 0x6b8, 0x7, 0x72, 0x2, 0x2, 0x6b7, 0x6b6, 0x3, 0x2, 0x2, 0x2, + 0x6b7, 0x6b8, 0x3, 0x2, 0x2, 0x2, 0x6b8, 0x6b9, 0x3, 0x2, 0x2, 0x2, + 0x6b9, 0x6bb, 0x9, 0x17, 0x2, 0x2, 0x6ba, 0x6a8, 0x3, 0x2, 0x2, 0x2, + 0x6ba, 0x6a9, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6aa, 0x3, 0x2, 0x2, 0x2, + 0x6ba, 0x6ab, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6ac, 0x3, 0x2, 0x2, 0x2, + 0x6ba, 0x6ad, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6ae, 0x3, 0x2, 0x2, 0x2, + 0x6ba, 0x6b0, 0x3, 0x2, 0x2, 0x2, 0x6ba, 0x6b7, 0x3, 0x2, 0x2, 0x2, + 0x6bb, 0x6bc, 0x3, 0x2, 0x2, 0x2, 0x6bc, 0x6e7, 0x5, 0xb0, 0x59, 0x11, + 0x6bd, 0x6be, 0xc, 0xd, 0x2, 0x2, 0x6be, 0x6bf, 0x7, 0x8, 0x2, 0x2, + 0x6bf, 0x6e7, 0x5, 0xb0, 0x59, 0xe, 0x6c0, 0x6c1, 0xc, 0xc, 0x2, 0x2, + 0x6c1, 0x6c2, 0x7, 0x78, 0x2, 0x2, 0x6c2, 0x6e7, 0x5, 0xb0, 0x59, 0xd, + 0x6c3, 0x6c5, 0xc, 0xb, 0x2, 0x2, 0x6c4, 0x6c6, 0x7, 0x72, 0x2, 0x2, + 0x6c5, 0x6c4, 0x3, 0x2, 0x2, 0x2, 0x6c5, 0x6c6, 0x3, 0x2, 0x2, 0x2, + 0x6c6, 0x6c7, 0x3, 0x2, 0x2, 0x2, 0x6c7, 0x6c8, 0x7, 0x12, 0x2, 0x2, + 0x6c8, 0x6c9, 0x5, 0xb0, 0x59, 0x2, 0x6c9, 0x6ca, 0x7, 0x8, 0x2, 0x2, + 0x6ca, 0x6cb, 0x5, 0xb0, 0x59, 0xc, 0x6cb, 0x6e7, 0x3, 0x2, 0x2, 0x2, + 0x6cc, 0x6cd, 0xc, 0xa, 0x2, 0x2, 0x6cd, 0x6ce, 0x7, 0xd5, 0x2, 0x2, + 0x6ce, 0x6cf, 0x5, 0xb0, 0x59, 0x2, 0x6cf, 0x6d0, 0x7, 0xc4, 0x2, 0x2, + 0x6d0, 0x6d1, 0x5, 0xb0, 0x59, 0xa, 0x6d1, 0x6e7, 0x3, 0x2, 0x2, 0x2, + 0x6d2, 0x6d3, 0xc, 0x15, 0x2, 0x2, 0x6d3, 0x6d4, 0x7, 0xce, 0x2, 0x2, + 0x6d4, 0x6d5, 0x5, 0xb0, 0x59, 0x2, 0x6d5, 0x6d6, 0x7, 0xd9, 0x2, 0x2, + 0x6d6, 0x6e7, 0x3, 0x2, 0x2, 0x2, 0x6d7, 0x6d8, 0xc, 0x14, 0x2, 0x2, + 0x6d8, 0x6d9, 0x7, 0xc8, 0x2, 0x2, 0x6d9, 0x6e7, 0x7, 0xbd, 0x2, 0x2, + 0x6da, 0x6db, 0xc, 0xf, 0x2, 0x2, 0x6db, 0x6dd, 0x7, 0x57, 0x2, 0x2, + 0x6dc, 0x6de, 0x7, 0x72, 0x2, 0x2, 0x6dd, 0x6dc, 0x3, 0x2, 0x2, 0x2, + 0x6dd, 0x6de, 0x3, 0x2, 0x2, 0x2, 0x6de, 0x6df, 0x3, 0x2, 0x2, 0x2, + 0x6df, 0x6e7, 0x7, 0x73, 0x2, 0x2, 0x6e0, 0x6e4, 0xc, 0x9, 0x2, 0x2, + 0x6e1, 0x6e5, 0x5, 0xd4, 0x6b, 0x2, 0x6e2, 0x6e3, 0x7, 0xc, 0x2, 0x2, + 0x6e3, 0x6e5, 0x5, 0xd6, 0x6c, 0x2, 0x6e4, 0x6e1, 0x3, 0x2, 0x2, 0x2, + 0x6e4, 0x6e2, 0x3, 0x2, 0x2, 0x2, 0x6e5, 0x6e7, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6a1, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6a4, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6a7, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6bd, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6c0, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6c3, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6cc, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6d2, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6d7, 0x3, 0x2, 0x2, 0x2, 0x6e6, 0x6da, 0x3, 0x2, 0x2, 0x2, + 0x6e6, 0x6e0, 0x3, 0x2, 0x2, 0x2, 0x6e7, 0x6ea, 0x3, 0x2, 0x2, 0x2, + 0x6e8, 0x6e6, 0x3, 0x2, 0x2, 0x2, 0x6e8, 0x6e9, 0x3, 0x2, 0x2, 0x2, + 0x6e9, 0xb1, 0x3, 0x2, 0x2, 0x2, 0x6ea, 0x6e8, 0x3, 0x2, 0x2, 0x2, 0x6eb, + 0x6f0, 0x5, 0xb4, 0x5b, 0x2, 0x6ec, 0x6ed, 0x7, 0xc5, 0x2, 0x2, 0x6ed, + 0x6ef, 0x5, 0xb4, 0x5b, 0x2, 0x6ee, 0x6ec, 0x3, 0x2, 0x2, 0x2, 0x6ef, + 0x6f2, 0x3, 0x2, 0x2, 0x2, 0x6f0, 0x6ee, 0x3, 0x2, 0x2, 0x2, 0x6f0, + 0x6f1, 0x3, 0x2, 0x2, 0x2, 0x6f1, 0xb3, 0x3, 0x2, 0x2, 0x2, 0x6f2, 0x6f0, + 0x3, 0x2, 0x2, 0x2, 0x6f3, 0x6f6, 0x5, 0xb6, 0x5c, 0x2, 0x6f4, 0x6f6, + 0x5, 0xb0, 0x59, 0x2, 0x6f5, 0x6f3, 0x3, 0x2, 0x2, 0x2, 0x6f5, 0x6f4, + 0x3, 0x2, 0x2, 0x2, 0x6f6, 0xb5, 0x3, 0x2, 0x2, 0x2, 0x6f7, 0x6f8, 0x7, + 0xd0, 0x2, 0x2, 0x6f8, 0x6fd, 0x5, 0xd6, 0x6c, 0x2, 0x6f9, 0x6fa, 0x7, + 0xc5, 0x2, 0x2, 0x6fa, 0x6fc, 0x5, 0xd6, 0x6c, 0x2, 0x6fb, 0x6f9, 0x3, + 0x2, 0x2, 0x2, 0x6fc, 0x6ff, 0x3, 0x2, 0x2, 0x2, 0x6fd, 0x6fb, 0x3, + 0x2, 0x2, 0x2, 0x6fd, 0x6fe, 0x3, 0x2, 0x2, 0x2, 0x6fe, 0x700, 0x3, + 0x2, 0x2, 0x2, 0x6ff, 0x6fd, 0x3, 0x2, 0x2, 0x2, 0x700, 0x701, 0x7, + 0xda, 0x2, 0x2, 0x701, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x702, 0x707, 0x5, + 0xd6, 0x6c, 0x2, 0x703, 0x704, 0x7, 0xc5, 0x2, 0x2, 0x704, 0x706, 0x5, + 0xd6, 0x6c, 0x2, 0x705, 0x703, 0x3, 0x2, 0x2, 0x2, 0x706, 0x709, 0x3, + 0x2, 0x2, 0x2, 0x707, 0x705, 0x3, 0x2, 0x2, 0x2, 0x707, 0x708, 0x3, + 0x2, 0x2, 0x2, 0x708, 0x70b, 0x3, 0x2, 0x2, 0x2, 0x709, 0x707, 0x3, + 0x2, 0x2, 0x2, 0x70a, 0x6f7, 0x3, 0x2, 0x2, 0x2, 0x70a, 0x702, 0x3, + 0x2, 0x2, 0x2, 0x70b, 0x70c, 0x3, 0x2, 0x2, 0x2, 0x70c, 0x70d, 0x7, + 0xc0, 0x2, 0x2, 0x70d, 0x70e, 0x5, 0xb0, 0x59, 0x2, 0x70e, 0xb7, 0x3, + 0x2, 0x2, 0x2, 0x70f, 0x710, 0x5, 0xc0, 0x61, 0x2, 0x710, 0x711, 0x7, + 0xc8, 0x2, 0x2, 0x711, 0x713, 0x3, 0x2, 0x2, 0x2, 0x712, 0x70f, 0x3, + 0x2, 0x2, 0x2, 0x712, 0x713, 0x3, 0x2, 0x2, 0x2, 0x713, 0x714, 0x3, + 0x2, 0x2, 0x2, 0x714, 0x715, 0x5, 0xba, 0x5e, 0x2, 0x715, 0xb9, 0x3, + 0x2, 0x2, 0x2, 0x716, 0x719, 0x5, 0xd6, 0x6c, 0x2, 0x717, 0x718, 0x7, + 0xc8, 0x2, 0x2, 0x718, 0x71a, 0x5, 0xd6, 0x6c, 0x2, 0x719, 0x717, 0x3, + 0x2, 0x2, 0x2, 0x719, 0x71a, 0x3, 0x2, 0x2, 0x2, 0x71a, 0xbb, 0x3, 0x2, + 0x2, 0x2, 0x71b, 0x71c, 0x8, 0x5f, 0x1, 0x2, 0x71c, 0x723, 0x5, 0xc0, + 0x61, 0x2, 0x71d, 0x723, 0x5, 0xbe, 0x60, 0x2, 0x71e, 0x71f, 0x7, 0xd0, + 0x2, 0x2, 0x71f, 0x720, 0x5, 0x68, 0x35, 0x2, 0x720, 0x721, 0x7, 0xda, + 0x2, 0x2, 0x721, 0x723, 0x3, 0x2, 0x2, 0x2, 0x722, 0x71b, 0x3, 0x2, + 0x2, 0x2, 0x722, 0x71d, 0x3, 0x2, 0x2, 0x2, 0x722, 0x71e, 0x3, 0x2, + 0x2, 0x2, 0x723, 0x72c, 0x3, 0x2, 0x2, 0x2, 0x724, 0x728, 0xc, 0x3, + 0x2, 0x2, 0x725, 0x729, 0x5, 0xd4, 0x6b, 0x2, 0x726, 0x727, 0x7, 0xc, + 0x2, 0x2, 0x727, 0x729, 0x5, 0xd6, 0x6c, 0x2, 0x728, 0x725, 0x3, 0x2, + 0x2, 0x2, 0x728, 0x726, 0x3, 0x2, 0x2, 0x2, 0x729, 0x72b, 0x3, 0x2, + 0x2, 0x2, 0x72a, 0x724, 0x3, 0x2, 0x2, 0x2, 0x72b, 0x72e, 0x3, 0x2, + 0x2, 0x2, 0x72c, 0x72a, 0x3, 0x2, 0x2, 0x2, 0x72c, 0x72d, 0x3, 0x2, + 0x2, 0x2, 0x72d, 0xbd, 0x3, 0x2, 0x2, 0x2, 0x72e, 0x72c, 0x3, 0x2, 0x2, + 0x2, 0x72f, 0x730, 0x5, 0xd6, 0x6c, 0x2, 0x730, 0x732, 0x7, 0xd0, 0x2, + 0x2, 0x731, 0x733, 0x5, 0xc2, 0x62, 0x2, 0x732, 0x731, 0x3, 0x2, 0x2, + 0x2, 0x732, 0x733, 0x3, 0x2, 0x2, 0x2, 0x733, 0x734, 0x3, 0x2, 0x2, + 0x2, 0x734, 0x735, 0x7, 0xda, 0x2, 0x2, 0x735, 0xbf, 0x3, 0x2, 0x2, + 0x2, 0x736, 0x737, 0x5, 0xc6, 0x64, 0x2, 0x737, 0x738, 0x7, 0xc8, 0x2, + 0x2, 0x738, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x739, 0x736, 0x3, 0x2, 0x2, + 0x2, 0x739, 0x73a, 0x3, 0x2, 0x2, 0x2, 0x73a, 0x73b, 0x3, 0x2, 0x2, + 0x2, 0x73b, 0x73c, 0x5, 0xd6, 0x6c, 0x2, 0x73c, 0xc1, 0x3, 0x2, 0x2, + 0x2, 0x73d, 0x742, 0x5, 0xc4, 0x63, 0x2, 0x73e, 0x73f, 0x7, 0xc5, 0x2, + 0x2, 0x73f, 0x741, 0x5, 0xc4, 0x63, 0x2, 0x740, 0x73e, 0x3, 0x2, 0x2, + 0x2, 0x741, 0x744, 0x3, 0x2, 0x2, 0x2, 0x742, 0x740, 0x3, 0x2, 0x2, + 0x2, 0x742, 0x743, 0x3, 0x2, 0x2, 0x2, 0x743, 0xc3, 0x3, 0x2, 0x2, 0x2, + 0x744, 0x742, 0x3, 0x2, 0x2, 0x2, 0x745, 0x749, 0x5, 0xd6, 0x6c, 0x2, + 0x746, 0x749, 0x5, 0xbe, 0x60, 0x2, 0x747, 0x749, 0x5, 0xcc, 0x67, 0x2, + 0x748, 0x745, 0x3, 0x2, 0x2, 0x2, 0x748, 0x746, 0x3, 0x2, 0x2, 0x2, + 0x748, 0x747, 0x3, 0x2, 0x2, 0x2, 0x749, 0xc5, 0x3, 0x2, 0x2, 0x2, 0x74a, + 0x74b, 0x5, 0xd6, 0x6c, 0x2, 0x74b, 0xc7, 0x3, 0x2, 0x2, 0x2, 0x74c, + 0x755, 0x7, 0xbb, 0x2, 0x2, 0x74d, 0x74e, 0x7, 0xc8, 0x2, 0x2, 0x74e, + 0x755, 0x9, 0x18, 0x2, 0x2, 0x74f, 0x750, 0x7, 0xbd, 0x2, 0x2, 0x750, + 0x752, 0x7, 0xc8, 0x2, 0x2, 0x751, 0x753, 0x9, 0x18, 0x2, 0x2, 0x752, + 0x751, 0x3, 0x2, 0x2, 0x2, 0x752, 0x753, 0x3, 0x2, 0x2, 0x2, 0x753, + 0x755, 0x3, 0x2, 0x2, 0x2, 0x754, 0x74c, 0x3, 0x2, 0x2, 0x2, 0x754, + 0x74d, 0x3, 0x2, 0x2, 0x2, 0x754, 0x74f, 0x3, 0x2, 0x2, 0x2, 0x755, + 0xc9, 0x3, 0x2, 0x2, 0x2, 0x756, 0x758, 0x9, 0x19, 0x2, 0x2, 0x757, + 0x756, 0x3, 0x2, 0x2, 0x2, 0x757, 0x758, 0x3, 0x2, 0x2, 0x2, 0x758, + 0x75f, 0x3, 0x2, 0x2, 0x2, 0x759, 0x760, 0x5, 0xc8, 0x65, 0x2, 0x75a, + 0x760, 0x7, 0xbc, 0x2, 0x2, 0x75b, 0x760, 0x7, 0xbd, 0x2, 0x2, 0x75c, + 0x760, 0x7, 0xbe, 0x2, 0x2, 0x75d, 0x760, 0x7, 0x51, 0x2, 0x2, 0x75e, + 0x760, 0x7, 0x70, 0x2, 0x2, 0x75f, 0x759, 0x3, 0x2, 0x2, 0x2, 0x75f, + 0x75a, 0x3, 0x2, 0x2, 0x2, 0x75f, 0x75b, 0x3, 0x2, 0x2, 0x2, 0x75f, + 0x75c, 0x3, 0x2, 0x2, 0x2, 0x75f, 0x75d, 0x3, 0x2, 0x2, 0x2, 0x75f, + 0x75e, 0x3, 0x2, 0x2, 0x2, 0x760, 0xcb, 0x3, 0x2, 0x2, 0x2, 0x761, 0x765, + 0x5, 0xca, 0x66, 0x2, 0x762, 0x765, 0x7, 0xbf, 0x2, 0x2, 0x763, 0x765, + 0x7, 0x73, 0x2, 0x2, 0x764, 0x761, 0x3, 0x2, 0x2, 0x2, 0x764, 0x762, + 0x3, 0x2, 0x2, 0x2, 0x764, 0x763, 0x3, 0x2, 0x2, 0x2, 0x765, 0xcd, 0x3, + 0x2, 0x2, 0x2, 0x766, 0x767, 0x9, 0x1a, 0x2, 0x2, 0x767, 0xcf, 0x3, + 0x2, 0x2, 0x2, 0x768, 0x769, 0x9, 0x1b, 0x2, 0x2, 0x769, 0xd1, 0x3, + 0x2, 0x2, 0x2, 0x76a, 0x76b, 0x9, 0x1c, 0x2, 0x2, 0x76b, 0xd3, 0x3, + 0x2, 0x2, 0x2, 0x76c, 0x76f, 0x7, 0xba, 0x2, 0x2, 0x76d, 0x76f, 0x5, + 0xd2, 0x6a, 0x2, 0x76e, 0x76c, 0x3, 0x2, 0x2, 0x2, 0x76e, 0x76d, 0x3, + 0x2, 0x2, 0x2, 0x76f, 0xd5, 0x3, 0x2, 0x2, 0x2, 0x770, 0x774, 0x7, 0xba, + 0x2, 0x2, 0x771, 0x774, 0x5, 0xce, 0x68, 0x2, 0x772, 0x774, 0x5, 0xd0, + 0x69, 0x2, 0x773, 0x770, 0x3, 0x2, 0x2, 0x2, 0x773, 0x771, 0x3, 0x2, + 0x2, 0x2, 0x773, 0x772, 0x3, 0x2, 0x2, 0x2, 0x774, 0xd7, 0x3, 0x2, 0x2, + 0x2, 0x775, 0x778, 0x5, 0xd6, 0x6c, 0x2, 0x776, 0x778, 0x7, 0x73, 0x2, + 0x2, 0x777, 0x775, 0x3, 0x2, 0x2, 0x2, 0x777, 0x776, 0x3, 0x2, 0x2, + 0x2, 0x778, 0xd9, 0x3, 0x2, 0x2, 0x2, 0x779, 0x77a, 0x7, 0xbf, 0x2, + 0x2, 0x77a, 0x77b, 0x7, 0xca, 0x2, 0x2, 0x77b, 0x77c, 0x5, 0xca, 0x66, + 0x2, 0x77c, 0xdb, 0x3, 0x2, 0x2, 0x2, 0x103, 0xe0, 0xe4, 0xe7, 0xea, + 0xfe, 0x104, 0x10b, 0x113, 0x118, 0x11f, 0x124, 0x12b, 0x130, 0x136, + 0x13c, 0x141, 0x147, 0x14c, 0x152, 0x157, 0x15d, 0x16b, 0x172, 0x179, + 0x180, 0x186, 0x18b, 0x191, 0x196, 0x19c, 0x1a5, 0x1af, 0x1b9, 0x1cd, + 0x1d5, 0x1e4, 0x1eb, 0x1f9, 0x1ff, 0x205, 0x20c, 0x210, 0x213, 0x21a, + 0x21e, 0x221, 0x22c, 0x230, 0x233, 0x238, 0x23a, 0x23d, 0x240, 0x24a, + 0x24e, 0x251, 0x254, 0x259, 0x25b, 0x261, 0x267, 0x26b, 0x26e, 0x271, + 0x274, 0x277, 0x27c, 0x282, 0x286, 0x289, 0x28c, 0x290, 0x298, 0x2b2, + 0x2b4, 0x2b8, 0x2ce, 0x2d0, 0x2db, 0x2de, 0x2e7, 0x2f8, 0x303, 0x315, + 0x322, 0x333, 0x33c, 0x357, 0x359, 0x36e, 0x373, 0x378, 0x37b, 0x387, + 0x38c, 0x390, 0x393, 0x397, 0x39b, 0x3a0, 0x3a3, 0x3a7, 0x3a9, 0x3bf, + 0x3c7, 0x3ca, 0x3d4, 0x3d8, 0x3e0, 0x3e4, 0x3e9, 0x3ed, 0x3f1, 0x3f5, + 0x3f9, 0x3fb, 0x403, 0x407, 0x40a, 0x412, 0x417, 0x41c, 0x41f, 0x429, + 0x433, 0x437, 0x43c, 0x440, 0x446, 0x449, 0x44c, 0x44f, 0x45d, 0x461, + 0x465, 0x46a, 0x46d, 0x477, 0x47f, 0x482, 0x486, 0x489, 0x48d, 0x490, + 0x493, 0x496, 0x499, 0x49d, 0x4a1, 0x4a4, 0x4a7, 0x4aa, 0x4ad, 0x4b0, + 0x4b9, 0x4bf, 0x4d3, 0x4e9, 0x4f1, 0x4f4, 0x4fa, 0x502, 0x505, 0x50b, + 0x50d, 0x511, 0x516, 0x519, 0x51c, 0x520, 0x524, 0x527, 0x529, 0x52c, + 0x530, 0x534, 0x537, 0x539, 0x53b, 0x53e, 0x543, 0x54e, 0x554, 0x559, + 0x560, 0x565, 0x569, 0x56d, 0x572, 0x579, 0x58e, 0x591, 0x59a, 0x59e, + 0x5a3, 0x5a8, 0x5ab, 0x5ad, 0x5c3, 0x5c6, 0x5d1, 0x5d5, 0x5d8, 0x5dc, + 0x5e0, 0x5e8, 0x5ec, 0x5f9, 0x605, 0x611, 0x619, 0x61d, 0x624, 0x62a, + 0x632, 0x637, 0x640, 0x644, 0x663, 0x674, 0x677, 0x67b, 0x67e, 0x68a, + 0x69b, 0x69f, 0x6b0, 0x6b3, 0x6b7, 0x6ba, 0x6c5, 0x6dd, 0x6e4, 0x6e6, + 0x6e8, 0x6f0, 0x6f5, 0x6fd, 0x707, 0x70a, 0x712, 0x719, 0x722, 0x728, + 0x72c, 0x732, 0x739, 0x742, 0x748, 0x752, 0x754, 0x757, 0x75f, 0x764, + 0x76e, 0x773, 0x777, }; atn::ATNDeserializer deserializer; diff --git a/src/Parsers/New/ClickHouseParser.g4 b/src/Parsers/New/ClickHouseParser.g4 index 8c237f96a5d..5cab874e329 100644 --- a/src/Parsers/New/ClickHouseParser.g4 +++ b/src/Parsers/New/ClickHouseParser.g4 @@ -193,7 +193,10 @@ existsStmt // EXPLAIN statement -explainStmt: EXPLAIN SYNTAX query; +explainStmt + : EXPLAIN AST query # ExplainASTStmt + | EXPLAIN SYNTAX query # ExplainSyntaxStmt + ; // INSERT statement @@ -425,7 +428,7 @@ tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; tableIdentifier: (databaseIdentifier DOT)? identifier; tableArgList: tableArgExpr (COMMA tableArgExpr)*; tableArgExpr - : tableIdentifier + : identifier | tableFunctionExpr | literal ; @@ -450,18 +453,18 @@ literal interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR; keyword // except NULL_SQL, INF, NAN_SQL - : AFTER | ALIAS | ALL | ALTER | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | ASYNC | ATTACH | BETWEEN | BOTH | BY | CASE | CAST - | CHECK | CLEAR | CLUSTER | CODEC | COLLATE | COLUMN | COMMENT | CONSTRAINT | CREATE | CROSS | CUBE | DATABASE | DATABASES | DATE - | DEDUPLICATE | DEFAULT | DELAY | DELETE | DESCRIBE | DESC | DESCENDING | DETACH | DICTIONARIES | DICTIONARY | DISK | DISTINCT + : AFTER | ALIAS | ALL | ALTER | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | AST | ASYNC | ATTACH | BETWEEN | BOTH | BY | CASE + | CAST | CHECK | CLEAR | CLUSTER | CODEC | COLLATE | COLUMN | COMMENT | CONSTRAINT | CREATE | CROSS | CUBE | DATABASE | DATABASES + | DATE | DEDUPLICATE | DEFAULT | DELAY | DELETE | DESCRIBE | DESC | DESCENDING | DETACH | DICTIONARIES | DICTIONARY | DISK | DISTINCT | DISTRIBUTED | DROP | ELSE | END | ENGINE | EVENTS | EXISTS | EXPLAIN | EXPRESSION | EXTRACT | FETCHES | FINAL | FIRST | FLUSH | FOR | FORMAT | FREEZE | FROM | FULL | FUNCTION | GLOBAL | GRANULARITY | GROUP | HAVING | HIERARCHICAL | ID | IF | ILIKE | IN | INDEX | INJECTIVE | INNER | INSERT | INTERVAL | INTO | IS | IS_OBJECT_ID | JOIN | JSON_FALSE | JSON_TRUE | KEY | KILL | LAST | LAYOUT - | LEADING | LEFT | LIFETIME | LIKE | LIMIT | LIVE | LOCAL | LOGS | MATERIALIZE | MATERIALIZED | MAX | MERGES | MIN | MODIFY | MOVE | MUTATION | NO - | NOT | NULLS | OFFSET | ON | OPTIMIZE | OR | ORDER | OUTER | OUTFILE | PARTITION | POPULATE | PREWHERE | PRIMARY | RANGE | RELOAD - | REMOVE | RENAME | REPLACE | REPLICA | REPLICATED | RIGHT | ROLLUP | SAMPLE | SELECT | SEMI | SENDS | SET | SETTINGS | SHOW | SOURCE - | START | STOP | SUBSTRING | SYNC | SYNTAX | SYSTEM | TABLE | TABLES | TEMPORARY | TEST | THEN | TIES | TIMEOUT | TIMESTAMP | TOTALS - | TRAILING | TRIM | TRUNCATE | TO | TOP | TTL | TYPE | UNION | UPDATE | USE | USING | UUID | VALUES | VIEW | VOLUME | WATCH | WHEN - | WHERE | WITH + | LEADING | LEFT | LIFETIME | LIKE | LIMIT | LIVE | LOCAL | LOGS | MATERIALIZE | MATERIALIZED | MAX | MERGES | MIN | MODIFY | MOVE + | MUTATION | NO | NOT | NULLS | OFFSET | ON | OPTIMIZE | OR | ORDER | OUTER | OUTFILE | PARTITION | POPULATE | PREWHERE | PRIMARY + | RANGE | RELOAD | REMOVE | RENAME | REPLACE | REPLICA | REPLICATED | RIGHT | ROLLUP | SAMPLE | SELECT | SEMI | SENDS | SET | SETTINGS + | SHOW | SOURCE | START | STOP | SUBSTRING | SYNC | SYNTAX | SYSTEM | TABLE | TABLES | TEMPORARY | TEST | THEN | TIES | TIMEOUT + | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TO | TOP | TTL | TYPE | UNION | UPDATE | USE | USING | UUID | VALUES | VIEW + | VOLUME | WATCH | WHEN | WHERE | WITH ; keywordForAlias : DATE | FIRST | ID | KEY diff --git a/src/Parsers/New/ClickHouseParser.h b/src/Parsers/New/ClickHouseParser.h index ecaff1a5add..8754559dcc2 100644 --- a/src/Parsers/New/ClickHouseParser.h +++ b/src/Parsers/New/ClickHouseParser.h @@ -14,46 +14,46 @@ class ClickHouseParser : public antlr4::Parser { public: enum { ADD = 1, AFTER = 2, ALIAS = 3, ALL = 4, ALTER = 5, AND = 6, ANTI = 7, - ANY = 8, ARRAY = 9, AS = 10, ASCENDING = 11, ASOF = 12, ASYNC = 13, - ATTACH = 14, BETWEEN = 15, BOTH = 16, BY = 17, CASE = 18, CAST = 19, - CHECK = 20, CLEAR = 21, CLUSTER = 22, CODEC = 23, COLLATE = 24, COLUMN = 25, - COMMENT = 26, CONSTRAINT = 27, CREATE = 28, CROSS = 29, CUBE = 30, DATABASE = 31, - DATABASES = 32, DATE = 33, DAY = 34, DEDUPLICATE = 35, DEFAULT = 36, - DELAY = 37, DELETE = 38, DESC = 39, DESCENDING = 40, DESCRIBE = 41, - DETACH = 42, DICTIONARIES = 43, DICTIONARY = 44, DISK = 45, DISTINCT = 46, - DISTRIBUTED = 47, DROP = 48, ELSE = 49, END = 50, ENGINE = 51, EVENTS = 52, - EXISTS = 53, EXPLAIN = 54, EXPRESSION = 55, EXTRACT = 56, FETCHES = 57, - FINAL = 58, FIRST = 59, FLUSH = 60, FOR = 61, FORMAT = 62, FREEZE = 63, - FROM = 64, FULL = 65, FUNCTION = 66, GLOBAL = 67, GRANULARITY = 68, - GROUP = 69, HAVING = 70, HIERARCHICAL = 71, HOUR = 72, ID = 73, IF = 74, - ILIKE = 75, IN = 76, INDEX = 77, INF = 78, INJECTIVE = 79, INNER = 80, - INSERT = 81, INTERVAL = 82, INTO = 83, IS = 84, IS_OBJECT_ID = 85, JOIN = 86, - KEY = 87, KILL = 88, LAST = 89, LAYOUT = 90, LEADING = 91, LEFT = 92, - LIFETIME = 93, LIKE = 94, LIMIT = 95, LIVE = 96, LOCAL = 97, LOGS = 98, - MATERIALIZED = 99, MATERIALIZE = 100, MAX = 101, MERGES = 102, MIN = 103, - MINUTE = 104, MODIFY = 105, MONTH = 106, MOVE = 107, MUTATION = 108, - NAN_SQL = 109, NO = 110, NOT = 111, NULL_SQL = 112, NULLS = 113, OFFSET = 114, - ON = 115, OPTIMIZE = 116, OR = 117, ORDER = 118, OUTER = 119, OUTFILE = 120, - PARTITION = 121, POPULATE = 122, PREWHERE = 123, PRIMARY = 124, PROJECTION = 125, - QUARTER = 126, RANGE = 127, RELOAD = 128, REMOVE = 129, RENAME = 130, - REPLACE = 131, REPLICA = 132, REPLICATED = 133, RIGHT = 134, ROLLUP = 135, - SAMPLE = 136, SECOND = 137, SELECT = 138, SEMI = 139, SENDS = 140, SET = 141, - SETTINGS = 142, SHOW = 143, SOURCE = 144, START = 145, STOP = 146, SUBSTRING = 147, - SYNC = 148, SYNTAX = 149, SYSTEM = 150, TABLE = 151, TABLES = 152, TEMPORARY = 153, - TEST = 154, THEN = 155, TIES = 156, TIMEOUT = 157, TIMESTAMP = 158, - TO = 159, TOP = 160, TOTALS = 161, TRAILING = 162, TRIM = 163, TRUNCATE = 164, - TTL = 165, TYPE = 166, UNION = 167, UPDATE = 168, USE = 169, USING = 170, - UUID = 171, VALUES = 172, VIEW = 173, VOLUME = 174, WATCH = 175, WEEK = 176, - WHEN = 177, WHERE = 178, WITH = 179, YEAR = 180, JSON_FALSE = 181, JSON_TRUE = 182, - IDENTIFIER = 183, FLOATING_LITERAL = 184, OCTAL_LITERAL = 185, DECIMAL_LITERAL = 186, - HEXADECIMAL_LITERAL = 187, STRING_LITERAL = 188, ARROW = 189, ASTERISK = 190, - BACKQUOTE = 191, BACKSLASH = 192, COLON = 193, COMMA = 194, CONCAT = 195, - DASH = 196, DOT = 197, EQ_DOUBLE = 198, EQ_SINGLE = 199, GE = 200, GT = 201, - LBRACE = 202, LBRACKET = 203, LE = 204, LPAREN = 205, LT = 206, NOT_EQ = 207, - PERCENT = 208, PLUS = 209, QUERY = 210, QUOTE_DOUBLE = 211, QUOTE_SINGLE = 212, - RBRACE = 213, RBRACKET = 214, RPAREN = 215, SEMICOLON = 216, SLASH = 217, - UNDERSCORE = 218, MULTI_LINE_COMMENT = 219, SINGLE_LINE_COMMENT = 220, - WHITESPACE = 221 + ANY = 8, ARRAY = 9, AS = 10, ASCENDING = 11, ASOF = 12, AST = 13, ASYNC = 14, + ATTACH = 15, BETWEEN = 16, BOTH = 17, BY = 18, CASE = 19, CAST = 20, + CHECK = 21, CLEAR = 22, CLUSTER = 23, CODEC = 24, COLLATE = 25, COLUMN = 26, + COMMENT = 27, CONSTRAINT = 28, CREATE = 29, CROSS = 30, CUBE = 31, DATABASE = 32, + DATABASES = 33, DATE = 34, DAY = 35, DEDUPLICATE = 36, DEFAULT = 37, + DELAY = 38, DELETE = 39, DESC = 40, DESCENDING = 41, DESCRIBE = 42, + DETACH = 43, DICTIONARIES = 44, DICTIONARY = 45, DISK = 46, DISTINCT = 47, + DISTRIBUTED = 48, DROP = 49, ELSE = 50, END = 51, ENGINE = 52, EVENTS = 53, + EXISTS = 54, EXPLAIN = 55, EXPRESSION = 56, EXTRACT = 57, FETCHES = 58, + FINAL = 59, FIRST = 60, FLUSH = 61, FOR = 62, FORMAT = 63, FREEZE = 64, + FROM = 65, FULL = 66, FUNCTION = 67, GLOBAL = 68, GRANULARITY = 69, + GROUP = 70, HAVING = 71, HIERARCHICAL = 72, HOUR = 73, ID = 74, IF = 75, + ILIKE = 76, IN = 77, INDEX = 78, INF = 79, INJECTIVE = 80, INNER = 81, + INSERT = 82, INTERVAL = 83, INTO = 84, IS = 85, IS_OBJECT_ID = 86, JOIN = 87, + KEY = 88, KILL = 89, LAST = 90, LAYOUT = 91, LEADING = 92, LEFT = 93, + LIFETIME = 94, LIKE = 95, LIMIT = 96, LIVE = 97, LOCAL = 98, LOGS = 99, + MATERIALIZE = 100, MATERIALIZED = 101, MAX = 102, MERGES = 103, MIN = 104, + MINUTE = 105, MODIFY = 106, MONTH = 107, MOVE = 108, MUTATION = 109, + NAN_SQL = 110, NO = 111, NOT = 112, NULL_SQL = 113, NULLS = 114, OFFSET = 115, + ON = 116, OPTIMIZE = 117, OR = 118, ORDER = 119, OUTER = 120, OUTFILE = 121, + PARTITION = 122, POPULATE = 123, PREWHERE = 124, PRIMARY = 125, PROJECTION = 126, + QUARTER = 127, RANGE = 128, RELOAD = 129, REMOVE = 130, RENAME = 131, + REPLACE = 132, REPLICA = 133, REPLICATED = 134, RIGHT = 135, ROLLUP = 136, + SAMPLE = 137, SECOND = 138, SELECT = 139, SEMI = 140, SENDS = 141, SET = 142, + SETTINGS = 143, SHOW = 144, SOURCE = 145, START = 146, STOP = 147, SUBSTRING = 148, + SYNC = 149, SYNTAX = 150, SYSTEM = 151, TABLE = 152, TABLES = 153, TEMPORARY = 154, + TEST = 155, THEN = 156, TIES = 157, TIMEOUT = 158, TIMESTAMP = 159, + TO = 160, TOP = 161, TOTALS = 162, TRAILING = 163, TRIM = 164, TRUNCATE = 165, + TTL = 166, TYPE = 167, UNION = 168, UPDATE = 169, USE = 170, USING = 171, + UUID = 172, VALUES = 173, VIEW = 174, VOLUME = 175, WATCH = 176, WEEK = 177, + WHEN = 178, WHERE = 179, WITH = 180, YEAR = 181, JSON_FALSE = 182, JSON_TRUE = 183, + IDENTIFIER = 184, FLOATING_LITERAL = 185, OCTAL_LITERAL = 186, DECIMAL_LITERAL = 187, + HEXADECIMAL_LITERAL = 188, STRING_LITERAL = 189, ARROW = 190, ASTERISK = 191, + BACKQUOTE = 192, BACKSLASH = 193, COLON = 194, COMMA = 195, CONCAT = 196, + DASH = 197, DOT = 198, EQ_DOUBLE = 199, EQ_SINGLE = 200, GE = 201, GT = 202, + LBRACE = 203, LBRACKET = 204, LE = 205, LPAREN = 206, LT = 207, NOT_EQ = 208, + PERCENT = 209, PLUS = 210, QUERY = 211, QUOTE_DOUBLE = 212, QUOTE_SINGLE = 213, + RBRACE = 214, RBRACKET = 215, RPAREN = 216, SEMICOLON = 217, SLASH = 218, + UNDERSCORE = 219, MULTI_LINE_COMMENT = 220, SINGLE_LINE_COMMENT = 221, + WHITESPACE = 222 }; enum { @@ -224,7 +224,6 @@ public: antlr4::tree::TerminalNode *SEMICOLON(); InsertStmtContext *insertStmt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -254,7 +253,6 @@ public: UseStmtContext *useStmt(); WatchStmtContext *watchStmt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -286,7 +284,6 @@ public: ClusterClauseContext *clusterClause(); std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -313,7 +310,6 @@ public: PartitionClauseContext *partitionClause(); antlr4::tree::TerminalNode *FROM(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -325,7 +321,6 @@ public: antlr4::tree::TerminalNode *ORDER(); antlr4::tree::TerminalNode *BY(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -336,7 +331,6 @@ public: antlr4::tree::TerminalNode *UPDATE(); AssignmentExprListContext *assignmentExprList(); WhereClauseContext *whereClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -351,7 +345,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *IN(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -366,7 +359,6 @@ public: TableColumnPropertyTypeContext *tableColumnPropertyType(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -377,7 +369,6 @@ public: antlr4::tree::TerminalNode *DELETE(); antlr4::tree::TerminalNode *WHERE(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -391,7 +382,6 @@ public: antlr4::tree::TerminalNode *STRING_LITERAL(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -404,7 +394,6 @@ public: NestedIdentifierContext *nestedIdentifier(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -414,7 +403,6 @@ public: antlr4::tree::TerminalNode *DETACH(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -430,7 +418,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *AFTER(); NestedIdentifierContext *nestedIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -440,7 +427,6 @@ public: antlr4::tree::TerminalNode *DROP(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -455,7 +441,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *IN(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -470,7 +455,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *IN(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -486,7 +470,6 @@ public: antlr4::tree::TerminalNode *VOLUME(); antlr4::tree::TerminalNode *TABLE(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -501,7 +484,6 @@ public: antlr4::tree::TerminalNode *TO(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -511,7 +493,6 @@ public: antlr4::tree::TerminalNode *FREEZE(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -526,7 +507,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *IN(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -539,7 +519,6 @@ public: TableColumnDfntContext *tableColumnDfnt(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -554,7 +533,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *IN(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -564,7 +542,6 @@ public: antlr4::tree::TerminalNode *REMOVE(); antlr4::tree::TerminalNode *TTL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -578,7 +555,6 @@ public: CodecExprContext *codecExpr(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -590,7 +566,6 @@ public: PartitionClauseContext *partitionClause(); antlr4::tree::TerminalNode *FROM(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -603,7 +578,6 @@ public: NestedIdentifierContext *nestedIdentifier(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -616,7 +590,6 @@ public: NestedIdentifierContext *nestedIdentifier(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -631,7 +604,6 @@ public: antlr4::tree::TerminalNode *STRING_LITERAL(); antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -641,7 +613,6 @@ public: antlr4::tree::TerminalNode *MODIFY(); TtlClauseContext *ttlClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -657,7 +628,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *AFTER(); NestedIdentifierContext *nestedIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -673,7 +643,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *AFTER(); NestedIdentifierContext *nestedIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -688,7 +657,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -703,7 +671,6 @@ public: antlr4::tree::TerminalNode *EQ_SINGLE(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -721,7 +688,6 @@ public: antlr4::tree::TerminalNode *MATERIALIZED(); antlr4::tree::TerminalNode *TTL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -737,7 +703,6 @@ public: antlr4::tree::TerminalNode *ID(); antlr4::tree::TerminalNode *STRING_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -765,7 +730,6 @@ public: antlr4::tree::TerminalNode *DICTIONARY(); TableIdentifierContext *tableIdentifier(); ClusterClauseContext *clusterClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -780,7 +744,6 @@ public: TableIdentifierContext *tableIdentifier(); PartitionClauseContext *partitionClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -817,7 +780,6 @@ public: UuidClauseContext *uuidClause(); ClusterClauseContext *clusterClause(); TableSchemaClauseContext *tableSchemaClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -836,7 +798,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); UuidClauseContext *uuidClause(); ClusterClauseContext *clusterClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -853,7 +814,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); ClusterClauseContext *clusterClause(); EngineExprContext *engineExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -877,7 +837,6 @@ public: DestinationClauseContext *destinationClause(); TableSchemaClauseContext *tableSchemaClause(); antlr4::tree::TerminalNode *DECIMAL_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -900,7 +859,6 @@ public: ClusterClauseContext *clusterClause(); TableSchemaClauseContext *tableSchemaClause(); antlr4::tree::TerminalNode *POPULATE(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -921,7 +879,6 @@ public: TableSchemaClauseContext *tableSchemaClause(); EngineClauseContext *engineClause(); SubqueryClauseContext *subqueryClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -938,7 +895,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -967,7 +923,6 @@ public: std::vector IS_OBJECT_ID(); antlr4::tree::TerminalNode* IS_OBJECT_ID(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -991,7 +946,6 @@ public: std::vector dictionarySettingsClause(); DictionarySettingsClauseContext* dictionarySettingsClause(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1006,7 +960,6 @@ public: antlr4::tree::TerminalNode *KEY(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1023,7 +976,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1043,7 +995,6 @@ public: std::vector dictionaryArgExpr(); DictionaryArgExprContext* dictionaryArgExpr(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1062,7 +1013,6 @@ public: antlr4::tree::TerminalNode *MIN(); antlr4::tree::TerminalNode *MAX(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1082,7 +1032,6 @@ public: std::vector dictionaryArgExpr(); DictionaryArgExprContext* dictionaryArgExpr(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1101,7 +1050,6 @@ public: IdentifierContext* identifier(size_t i); antlr4::tree::TerminalNode *MAX(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1117,7 +1065,6 @@ public: SettingExprListContext *settingExprList(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1133,7 +1080,6 @@ public: IdentifierContext *identifier(); antlr4::tree::TerminalNode *STRING_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1147,7 +1093,6 @@ public: antlr4::tree::TerminalNode *UUID(); antlr4::tree::TerminalNode *STRING_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1161,7 +1106,6 @@ public: antlr4::tree::TerminalNode *TO(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1175,7 +1119,6 @@ public: antlr4::tree::TerminalNode *AS(); SelectUnionStmtContext *selectUnionStmt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1201,7 +1144,6 @@ public: antlr4::tree::TerminalNode *AS(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1211,7 +1153,6 @@ public: antlr4::tree::TerminalNode *AS(); TableFunctionExprContext *tableFunctionExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1225,7 +1166,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1250,7 +1190,6 @@ public: std::vector settingsClause(); SettingsClauseContext* settingsClause(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1265,7 +1204,6 @@ public: antlr4::tree::TerminalNode *BY(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1280,7 +1218,6 @@ public: antlr4::tree::TerminalNode *KEY(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1295,7 +1232,6 @@ public: antlr4::tree::TerminalNode *BY(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1312,7 +1248,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1330,7 +1265,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1356,7 +1290,6 @@ public: antlr4::tree::TerminalNode *PROJECTION(); TableProjectionDfntContext *tableProjectionDfnt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1368,7 +1301,6 @@ public: IdentifierContext *identifier(); antlr4::tree::TerminalNode *CHECK(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1377,7 +1309,6 @@ public: TableElementExprColumnContext(TableElementExprContext *ctx); TableColumnDfntContext *tableColumnDfnt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1387,7 +1318,6 @@ public: antlr4::tree::TerminalNode *INDEX(); TableIndexDfntContext *tableIndexDfnt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1406,7 +1336,6 @@ public: antlr4::tree::TerminalNode *TTL(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1422,7 +1351,6 @@ public: antlr4::tree::TerminalNode *MATERIALIZED(); antlr4::tree::TerminalNode *ALIAS(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1440,7 +1368,6 @@ public: antlr4::tree::TerminalNode *GRANULARITY(); antlr4::tree::TerminalNode *DECIMAL_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1454,7 +1381,6 @@ public: NestedIdentifierContext *nestedIdentifier(); ProjectionSelectStmtContext *projectionSelectStmt(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1473,7 +1399,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1489,7 +1414,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1507,7 +1431,6 @@ public: antlr4::tree::TerminalNode *STRING_LITERAL(); antlr4::tree::TerminalNode *VOLUME(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1523,7 +1446,6 @@ public: antlr4::tree::TerminalNode *DESC(); antlr4::tree::TerminalNode *TABLE(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1554,7 +1476,6 @@ public: antlr4::tree::TerminalNode *IF(); antlr4::tree::TerminalNode *EXISTS(); ClusterClauseContext *clusterClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1574,7 +1495,6 @@ public: antlr4::tree::TerminalNode *NO(); antlr4::tree::TerminalNode *DELAY(); antlr4::tree::TerminalNode *TEMPORARY(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1603,7 +1523,6 @@ public: antlr4::tree::TerminalNode *TABLE(); antlr4::tree::TerminalNode *VIEW(); antlr4::tree::TerminalNode *TEMPORARY(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1614,7 +1533,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); antlr4::tree::TerminalNode *DATABASE(); DatabaseIdentifierContext *databaseIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1623,14 +1541,34 @@ public: class ExplainStmtContext : public antlr4::ParserRuleContext { public: ExplainStmtContext(antlr4::ParserRuleContext *parent, size_t invokingState); + + ExplainStmtContext() = default; + void copyFrom(ExplainStmtContext *context); + using antlr4::ParserRuleContext::copyFrom; + virtual size_t getRuleIndex() const override; + + + }; + + class ExplainSyntaxStmtContext : public ExplainStmtContext { + public: + ExplainSyntaxStmtContext(ExplainStmtContext *ctx); + antlr4::tree::TerminalNode *EXPLAIN(); antlr4::tree::TerminalNode *SYNTAX(); QueryContext *query(); - - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; - + }; + + class ExplainASTStmtContext : public ExplainStmtContext { + public: + ExplainASTStmtContext(ExplainStmtContext *ctx); + + antlr4::tree::TerminalNode *EXPLAIN(); + antlr4::tree::TerminalNode *AST(); + QueryContext *query(); + virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; ExplainStmtContext* explainStmt(); @@ -1648,7 +1586,6 @@ public: antlr4::tree::TerminalNode *TABLE(); ColumnsClauseContext *columnsClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1666,7 +1603,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1691,7 +1627,6 @@ public: DataClauseValuesContext(DataClauseContext *ctx); antlr4::tree::TerminalNode *VALUES(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1701,7 +1636,6 @@ public: antlr4::tree::TerminalNode *FORMAT(); IdentifierContext *identifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1712,7 +1646,6 @@ public: SelectUnionStmtContext *selectUnionStmt(); antlr4::tree::TerminalNode *EOF(); antlr4::tree::TerminalNode *SEMICOLON(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1742,7 +1675,6 @@ public: antlr4::tree::TerminalNode *SYNC(); antlr4::tree::TerminalNode *ASYNC(); antlr4::tree::TerminalNode *TEST(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1760,7 +1692,6 @@ public: antlr4::tree::TerminalNode *FINAL(); antlr4::tree::TerminalNode *DEDUPLICATE(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1781,7 +1712,6 @@ public: antlr4::tree::TerminalNode* COMMA(size_t i); ClusterClauseContext *clusterClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1800,7 +1730,6 @@ public: GroupByClauseContext *groupByClause(); ProjectionOrderByClauseContext *projectionOrderByClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1818,7 +1747,6 @@ public: std::vector ALL(); antlr4::tree::TerminalNode* ALL(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1834,7 +1762,6 @@ public: SelectUnionStmtContext *selectUnionStmt(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1866,7 +1793,6 @@ public: antlr4::tree::TerminalNode *CUBE(); antlr4::tree::TerminalNode *ROLLUP(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1880,7 +1806,6 @@ public: antlr4::tree::TerminalNode *WITH(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1896,7 +1821,6 @@ public: antlr4::tree::TerminalNode *WITH(); antlr4::tree::TerminalNode *TIES(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1910,7 +1834,6 @@ public: antlr4::tree::TerminalNode *FROM(); JoinExprContext *joinExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1927,7 +1850,6 @@ public: antlr4::tree::TerminalNode *LEFT(); antlr4::tree::TerminalNode *INNER(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1941,7 +1863,6 @@ public: antlr4::tree::TerminalNode *PREWHERE(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1955,7 +1876,6 @@ public: antlr4::tree::TerminalNode *WHERE(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1974,7 +1894,6 @@ public: antlr4::tree::TerminalNode *CUBE(); antlr4::tree::TerminalNode *ROLLUP(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -1988,7 +1907,6 @@ public: antlr4::tree::TerminalNode *HAVING(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2003,7 +1921,6 @@ public: antlr4::tree::TerminalNode *BY(); OrderExprListContext *orderExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2018,7 +1935,6 @@ public: antlr4::tree::TerminalNode *BY(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2034,7 +1950,6 @@ public: antlr4::tree::TerminalNode *BY(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2050,7 +1965,6 @@ public: antlr4::tree::TerminalNode *WITH(); antlr4::tree::TerminalNode *TIES(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2064,7 +1978,6 @@ public: antlr4::tree::TerminalNode *SETTINGS(); SettingExprListContext *settingExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2095,7 +2008,6 @@ public: JoinOpContext *joinOp(); antlr4::tree::TerminalNode *GLOBAL(); antlr4::tree::TerminalNode *LOCAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2106,7 +2018,6 @@ public: TableExprContext *tableExpr(); antlr4::tree::TerminalNode *FINAL(); SampleClauseContext *sampleClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2117,7 +2028,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); JoinExprContext *joinExpr(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2128,7 +2038,6 @@ public: std::vector joinExpr(); JoinExprContext* joinExpr(size_t i); JoinOpCrossContext *joinOpCross(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2155,7 +2064,6 @@ public: antlr4::tree::TerminalNode *OUTER(); antlr4::tree::TerminalNode *ALL(); antlr4::tree::TerminalNode *ANY(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2167,7 +2075,6 @@ public: antlr4::tree::TerminalNode *ALL(); antlr4::tree::TerminalNode *ANY(); antlr4::tree::TerminalNode *ASOF(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2183,7 +2090,6 @@ public: antlr4::tree::TerminalNode *ANTI(); antlr4::tree::TerminalNode *ANY(); antlr4::tree::TerminalNode *ASOF(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2199,7 +2105,6 @@ public: antlr4::tree::TerminalNode *LOCAL(); antlr4::tree::TerminalNode *COMMA(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2216,7 +2121,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2232,7 +2136,6 @@ public: RatioExprContext* ratioExpr(size_t i); antlr4::tree::TerminalNode *OFFSET(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2248,7 +2151,6 @@ public: antlr4::tree::TerminalNode *COMMA(); antlr4::tree::TerminalNode *OFFSET(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2264,7 +2166,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2285,7 +2186,6 @@ public: antlr4::tree::TerminalNode *FIRST(); antlr4::tree::TerminalNode *LAST(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2300,7 +2200,6 @@ public: NumberLiteralContext* numberLiteral(size_t i); antlr4::tree::TerminalNode *SLASH(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2316,7 +2215,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2331,7 +2229,6 @@ public: antlr4::tree::TerminalNode *EQ_SINGLE(); LiteralContext *literal(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2345,7 +2242,6 @@ public: antlr4::tree::TerminalNode *SET(); SettingExprListContext *settingExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2373,7 +2269,6 @@ public: antlr4::tree::TerminalNode *CREATE(); antlr4::tree::TerminalNode *DATABASE(); DatabaseIdentifierContext *databaseIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2383,7 +2278,6 @@ public: antlr4::tree::TerminalNode *SHOW(); antlr4::tree::TerminalNode *DATABASES(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2396,7 +2290,6 @@ public: TableIdentifierContext *tableIdentifier(); antlr4::tree::TerminalNode *TEMPORARY(); antlr4::tree::TerminalNode *TABLE(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2414,7 +2307,6 @@ public: LimitClauseContext *limitClause(); antlr4::tree::TerminalNode *FROM(); antlr4::tree::TerminalNode *IN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2426,7 +2318,6 @@ public: antlr4::tree::TerminalNode *DICTIONARIES(); antlr4::tree::TerminalNode *FROM(); DatabaseIdentifierContext *databaseIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2438,7 +2329,6 @@ public: antlr4::tree::TerminalNode *CREATE(); antlr4::tree::TerminalNode *DICTIONARY(); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2466,7 +2356,6 @@ public: antlr4::tree::TerminalNode *SYNC(); antlr4::tree::TerminalNode *REPLICA(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2485,7 +2374,6 @@ public: antlr4::tree::TerminalNode *EXISTS(); ClusterClauseContext *clusterClause(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2499,7 +2387,6 @@ public: antlr4::tree::TerminalNode *USE(); DatabaseIdentifierContext *databaseIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2516,7 +2403,6 @@ public: antlr4::tree::TerminalNode *LIMIT(); antlr4::tree::TerminalNode *DECIMAL_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2548,7 +2434,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2560,7 +2445,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); antlr4::tree::TerminalNode *RPAREN(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2569,7 +2453,6 @@ public: ColumnTypeExprSimpleContext(ColumnTypeExprContext *ctx); IdentifierContext *identifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2584,7 +2467,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2599,7 +2481,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2614,7 +2495,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2639,7 +2519,6 @@ public: ColumnsExprColumnContext(ColumnsExprContext *ctx); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2650,7 +2529,6 @@ public: antlr4::tree::TerminalNode *ASTERISK(); TableIdentifierContext *tableIdentifier(); antlr4::tree::TerminalNode *DOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2661,7 +2539,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); SelectUnionStmtContext *selectUnionStmt(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2688,7 +2565,6 @@ public: ColumnExprContext* columnExpr(size_t i); antlr4::tree::TerminalNode *QUERY(); antlr4::tree::TerminalNode *COLON(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2700,7 +2576,6 @@ public: AliasContext *alias(); antlr4::tree::TerminalNode *AS(); IdentifierContext *identifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2714,7 +2589,6 @@ public: antlr4::tree::TerminalNode *FROM(); ColumnExprContext *columnExpr(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2724,7 +2598,6 @@ public: antlr4::tree::TerminalNode *DASH(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2735,7 +2608,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); SelectUnionStmtContext *selectUnionStmt(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2744,7 +2616,6 @@ public: ColumnExprLiteralContext(ColumnExprContext *ctx); LiteralContext *literal(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2755,7 +2626,6 @@ public: antlr4::tree::TerminalNode *LBRACKET(); antlr4::tree::TerminalNode *RBRACKET(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2770,7 +2640,6 @@ public: antlr4::tree::TerminalNode *FROM(); antlr4::tree::TerminalNode *RPAREN(); antlr4::tree::TerminalNode *FOR(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2784,7 +2653,6 @@ public: antlr4::tree::TerminalNode *AS(); ColumnTypeExprContext *columnTypeExpr(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2795,7 +2663,6 @@ public: std::vector columnExpr(); ColumnExprContext* columnExpr(size_t i); antlr4::tree::TerminalNode *OR(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2808,7 +2675,6 @@ public: antlr4::tree::TerminalNode *ASTERISK(); antlr4::tree::TerminalNode *SLASH(); antlr4::tree::TerminalNode *PERCENT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2821,7 +2687,6 @@ public: antlr4::tree::TerminalNode *PLUS(); antlr4::tree::TerminalNode *DASH(); antlr4::tree::TerminalNode *CONCAT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2843,7 +2708,6 @@ public: antlr4::tree::TerminalNode *ILIKE(); antlr4::tree::TerminalNode *GLOBAL(); antlr4::tree::TerminalNode *NOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2854,7 +2718,6 @@ public: antlr4::tree::TerminalNode *INTERVAL(); ColumnExprContext *columnExpr(); IntervalContext *interval(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2866,7 +2729,6 @@ public: antlr4::tree::TerminalNode *IS(); antlr4::tree::TerminalNode *NULL_SQL(); antlr4::tree::TerminalNode *NOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2883,7 +2745,6 @@ public: antlr4::tree::TerminalNode *BOTH(); antlr4::tree::TerminalNode *LEADING(); antlr4::tree::TerminalNode *TRAILING(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2894,7 +2755,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); ColumnExprListContext *columnExprList(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2906,7 +2766,6 @@ public: ColumnExprContext* columnExpr(size_t i); antlr4::tree::TerminalNode *LBRACKET(); antlr4::tree::TerminalNode *RBRACKET(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2919,7 +2778,6 @@ public: antlr4::tree::TerminalNode *BETWEEN(); antlr4::tree::TerminalNode *AND(); antlr4::tree::TerminalNode *NOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2930,7 +2788,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); ColumnExprContext *columnExpr(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2940,7 +2797,6 @@ public: antlr4::tree::TerminalNode *TIMESTAMP(); antlr4::tree::TerminalNode *STRING_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2951,7 +2807,6 @@ public: std::vector columnExpr(); ColumnExprContext* columnExpr(size_t i); antlr4::tree::TerminalNode *AND(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2962,7 +2817,6 @@ public: ColumnExprContext *columnExpr(); antlr4::tree::TerminalNode *DOT(); antlr4::tree::TerminalNode *DECIMAL_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2979,7 +2833,6 @@ public: std::vector THEN(); antlr4::tree::TerminalNode* THEN(size_t i); antlr4::tree::TerminalNode *ELSE(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2989,7 +2842,6 @@ public: antlr4::tree::TerminalNode *DATE(); antlr4::tree::TerminalNode *STRING_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -2999,7 +2851,6 @@ public: antlr4::tree::TerminalNode *NOT(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3008,7 +2859,6 @@ public: ColumnExprIdentifierContext(ColumnExprContext *ctx); ColumnIdentifierContext *columnIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3024,7 +2874,6 @@ public: antlr4::tree::TerminalNode *DISTINCT(); ColumnArgListContext *columnArgList(); ColumnExprListContext *columnExprList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3035,7 +2884,6 @@ public: antlr4::tree::TerminalNode *ASTERISK(); TableIdentifierContext *tableIdentifier(); antlr4::tree::TerminalNode *DOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3050,7 +2898,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3064,7 +2911,6 @@ public: ColumnLambdaExprContext *columnLambdaExpr(); ColumnExprContext *columnExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3084,7 +2930,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3099,7 +2944,6 @@ public: TableIdentifierContext *tableIdentifier(); antlr4::tree::TerminalNode *DOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3114,7 +2958,6 @@ public: IdentifierContext* identifier(size_t i); antlr4::tree::TerminalNode *DOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3139,7 +2982,6 @@ public: TableExprIdentifierContext(TableExprContext *ctx); TableIdentifierContext *tableIdentifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3150,7 +2992,6 @@ public: antlr4::tree::TerminalNode *LPAREN(); SelectUnionStmtContext *selectUnionStmt(); antlr4::tree::TerminalNode *RPAREN(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3162,7 +3003,6 @@ public: AliasContext *alias(); antlr4::tree::TerminalNode *AS(); IdentifierContext *identifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3171,7 +3011,6 @@ public: TableExprFunctionContext(TableExprContext *ctx); TableFunctionExprContext *tableFunctionExpr(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3186,7 +3025,6 @@ public: antlr4::tree::TerminalNode *RPAREN(); TableArgListContext *tableArgList(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3201,7 +3039,6 @@ public: DatabaseIdentifierContext *databaseIdentifier(); antlr4::tree::TerminalNode *DOT(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3217,7 +3054,6 @@ public: std::vector COMMA(); antlr4::tree::TerminalNode* COMMA(size_t i); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3228,11 +3064,10 @@ public: public: TableArgExprContext(antlr4::ParserRuleContext *parent, size_t invokingState); virtual size_t getRuleIndex() const override; - TableIdentifierContext *tableIdentifier(); + IdentifierContext *identifier(); TableFunctionExprContext *tableFunctionExpr(); LiteralContext *literal(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3245,7 +3080,6 @@ public: virtual size_t getRuleIndex() const override; IdentifierContext *identifier(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3262,7 +3096,6 @@ public: antlr4::tree::TerminalNode* DECIMAL_LITERAL(size_t i); antlr4::tree::TerminalNode *OCTAL_LITERAL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3282,7 +3115,6 @@ public: antlr4::tree::TerminalNode *PLUS(); antlr4::tree::TerminalNode *DASH(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3297,7 +3129,6 @@ public: antlr4::tree::TerminalNode *STRING_LITERAL(); antlr4::tree::TerminalNode *NULL_SQL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3317,7 +3148,6 @@ public: antlr4::tree::TerminalNode *QUARTER(); antlr4::tree::TerminalNode *YEAR(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3339,6 +3169,7 @@ public: antlr4::tree::TerminalNode *AS(); antlr4::tree::TerminalNode *ASCENDING(); antlr4::tree::TerminalNode *ASOF(); + antlr4::tree::TerminalNode *AST(); antlr4::tree::TerminalNode *ASYNC(); antlr4::tree::TerminalNode *ATTACH(); antlr4::tree::TerminalNode *BETWEEN(); @@ -3498,7 +3329,6 @@ public: antlr4::tree::TerminalNode *WHERE(); antlr4::tree::TerminalNode *WITH(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3514,7 +3344,6 @@ public: antlr4::tree::TerminalNode *ID(); antlr4::tree::TerminalNode *KEY(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3528,7 +3357,6 @@ public: antlr4::tree::TerminalNode *IDENTIFIER(); KeywordForAliasContext *keywordForAlias(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3543,7 +3371,6 @@ public: IntervalContext *interval(); KeywordContext *keyword(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3557,7 +3384,6 @@ public: IdentifierContext *identifier(); antlr4::tree::TerminalNode *NULL_SQL(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; @@ -3572,7 +3398,6 @@ public: antlr4::tree::TerminalNode *EQ_SINGLE(); NumberLiteralContext *numberLiteral(); - virtual antlrcpp::Any accept(antlr4::tree::ParseTreeVisitor *visitor) override; }; diff --git a/src/Parsers/New/ClickHouseParserVisitor.h b/src/Parsers/New/ClickHouseParserVisitor.h index 0cb1876f556..088fdd7f0ca 100644 --- a/src/Parsers/New/ClickHouseParserVisitor.h +++ b/src/Parsers/New/ClickHouseParserVisitor.h @@ -184,7 +184,9 @@ public: virtual antlrcpp::Any visitExistsTableStmt(ClickHouseParser::ExistsTableStmtContext *context) = 0; - virtual antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext *context) = 0; + virtual antlrcpp::Any visitExplainASTStmt(ClickHouseParser::ExplainASTStmtContext *context) = 0; + + virtual antlrcpp::Any visitExplainSyntaxStmt(ClickHouseParser::ExplainSyntaxStmtContext *context) = 0; virtual antlrcpp::Any visitInsertStmt(ClickHouseParser::InsertStmtContext *context) = 0; diff --git a/src/Parsers/New/ParseTreeVisitor.h b/src/Parsers/New/ParseTreeVisitor.h index 3b07c1beed8..35d5ae9b12e 100644 --- a/src/Parsers/New/ParseTreeVisitor.h +++ b/src/Parsers/New/ParseTreeVisitor.h @@ -146,7 +146,8 @@ public: antlrcpp::Any visitExistsDatabaseStmt(ClickHouseParser::ExistsDatabaseStmtContext * ctx) override; // ExplainQuery - antlrcpp::Any visitExplainStmt(ClickHouseParser::ExplainStmtContext * ctx) override; + antlrcpp::Any visitExplainASTStmt(ClickHouseParser::ExplainASTStmtContext * ctx) override; + antlrcpp::Any visitExplainSyntaxStmt(ClickHouseParser::ExplainSyntaxStmtContext * ctx) override; // Identifier antlrcpp::Any visitTableIdentifier(ClickHouseParser::TableIdentifierContext * ctx) override; diff --git a/tests/queries/conftest.py b/tests/queries/conftest.py index 40a9a6b3a2e..f9659f2f98b 100644 --- a/tests/queries/conftest.py +++ b/tests/queries/conftest.py @@ -1,22 +1,21 @@ -import pytest - import os import sys -import tempfile + +import pytest from .server import ServerThread def pytest_addoption(parser): - parser.addoption( - "--builddir", action="store", default=None, help="Path to build directory to use binaries from", - ) + parser.addoption("--builddir", action="store", default=None, help="Path to build directory to use binaries from") + parser.addoption("--antlr", action="store_true", default=False, help="Use ANTLR parser") @pytest.fixture(scope='module') def cmdopts(request): return { 'builddir': request.config.getoption("--builddir"), + 'antlr': request.config.getoption("--antlr"), } @@ -31,6 +30,11 @@ def bin_prefix(cmdopts): return prefix +@pytest.fixture(scope='module') +def use_antlr(cmdopts): + return cmdopts['antlr'] + + # TODO: also support stateful queries. QUERIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '0_stateless') diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index ed178053326..2d549301cb0 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -137,17 +137,19 @@ def check_result(result, error, return_code, reference, replace_map): pytrace=False) -def run_client(bin_prefix, port, database, query, reference, replace_map=None): +def run_client(use_antlr, bin_prefix, port, database, query, reference, replace_map=None): # We can't use `text=True` since some tests may return binary data - client = subprocess.Popen([bin_prefix + '-client', '--port', str(port), '-d', database, '-m', '-n', '--testmode'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = [bin_prefix + '-client', '--port', str(port), '-d', database, '-m', '-n', '--testmode'] + if use_antlr: + cmd.append('--use_antlr_parser=1') + client = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) result, error = client.communicate(query.encode('utf-8')) assert client.returncode is not None, "Client should exit after processing all queries" check_result(result, error, client.returncode, reference, replace_map) -def run_shell(bin_prefix, server, database, path, reference, replace_map=None): +def run_shell(use_antlr, bin_prefix, server, database, path, reference, replace_map=None): env = { 'CLICKHOUSE_BINARY': bin_prefix, 'CLICKHOUSE_DATABASE': database, @@ -160,6 +162,8 @@ def run_shell(bin_prefix, server, database, path, reference, replace_map=None): 'CLICKHOUSE_CONFIG_CLIENT': server.client_config, 'PROTOC_BINARY': os.path.abspath(os.path.join(os.path.dirname(bin_prefix), '..', 'contrib', 'protobuf', 'protoc')), # FIXME: adhoc solution } + if use_antlr: + env['CLICKHOUSE_CLIENT_OPT'] = '--use_antlr_parser=1' shell = subprocess.Popen([path], env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) result, error = shell.communicate() assert shell.returncode is not None, "Script should exit after executing all commands" @@ -173,7 +177,7 @@ def random_str(length=10): return ''.join(random.choice(alphabet) for _ in range(length)) -def test_sql_query(bin_prefix, sql_query, standalone_server): +def test_sql_query(use_antlr, bin_prefix, sql_query, standalone_server): for test in SKIP_LIST: if test in sql_query: pytest.skip("Test matches skip-list: " + test) @@ -193,21 +197,21 @@ def test_sql_query(bin_prefix, sql_query, standalone_server): reference = file.read() random_name = 'test_{random}'.format(random=random_str()) - run_client(bin_prefix, tcp_port, 'default', 'CREATE DATABASE {random};'.format(random=random_name), b'') + run_client(use_antlr, bin_prefix, tcp_port, 'default', 'CREATE DATABASE {random};'.format(random=random_name), b'') - run_client(bin_prefix, tcp_port, random_name, query, reference, {random_name: 'default'}) + run_client(use_antlr, bin_prefix, tcp_port, random_name, query, reference, {random_name: 'default'}) query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);" - run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n') query = 'DROP DATABASE {random};'.format(random=random_name) - run_client(bin_prefix, tcp_port, 'default', query, b'') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'') query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;" - run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n') -def test_shell_query(bin_prefix, shell_query, standalone_server): +def test_shell_query(use_antlr, bin_prefix, shell_query, standalone_server): for test in SKIP_LIST: if test in shell_query: pytest.skip("Test matches skip-list: " + test) @@ -226,15 +230,15 @@ def test_shell_query(bin_prefix, shell_query, standalone_server): random_name = 'test_{random}'.format(random=random_str()) query = 'CREATE DATABASE {random};'.format(random=random_name) - run_client(bin_prefix, tcp_port, 'default', query, b'') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'') - run_shell(bin_prefix, standalone_server, random_name, shell_path, reference, {random_name: 'default'}) + run_shell(use_antlr, bin_prefix, standalone_server, random_name, shell_path, reference, {random_name: 'default'}) query = "SELECT 'SHOW ORPHANED TABLES'; SELECT name FROM system.tables WHERE database != 'system' ORDER BY (database, name);" - run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED TABLES\n') query = 'DROP DATABASE {random};'.format(random=random_name) - run_client(bin_prefix, tcp_port, 'default', query, b'') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'') query = "SELECT 'SHOW ORPHANED DATABASES'; SHOW DATABASES;" - run_client(bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n') + run_client(use_antlr, bin_prefix, tcp_port, 'default', query, b'SHOW ORPHANED DATABASES\ndefault\nsystem\n') From e6b9ab62618281257698369f0eebb349f300ea2d Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Fri, 11 Jun 2021 21:05:45 +0300 Subject: [PATCH 105/491] Fix false-positive issue from clang-tidy --- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index dc02057f560..11157410501 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -89,7 +89,8 @@ void ApplyWithSubqueryVisitor::visit(ASTFunction & func, const Data & data) { if (identifier->isShort()) { - const auto & name = identifier->shortName(); + /// Clang-tidy is wrong on this line, because `func.arguments->children.at(1)` gets replaced before last use of `name`. + auto name = identifier->shortName(); // NOLINT auto subquery_it = data.subqueries.find(name); if (subquery_it != data.subqueries.end()) { From a237229998931169851321ec2d5af405429055a9 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Sat, 12 Jun 2021 00:15:38 +0300 Subject: [PATCH 106/491] Another ANTLR fix --- src/Parsers/New/AST/TableExpr.cpp | 2 +- src/Parsers/New/ClickHouseParser.cpp | 8 ++++---- src/Parsers/New/ClickHouseParser.g4 | 2 +- src/Parsers/New/ClickHouseParser.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Parsers/New/AST/TableExpr.cpp b/src/Parsers/New/AST/TableExpr.cpp index 63768e16f53..e14493c6bd6 100644 --- a/src/Parsers/New/AST/TableExpr.cpp +++ b/src/Parsers/New/AST/TableExpr.cpp @@ -149,7 +149,7 @@ antlrcpp::Any ParseTreeVisitor::visitTableArgExpr(ClickHouseParser::TableArgExpr { if (ctx->literal()) return std::make_shared(visit(ctx->literal()).as>()); if (ctx->tableFunctionExpr()) return std::make_shared(visit(ctx->tableFunctionExpr()).as>()); - if (ctx->identifier()) return std::make_shared(visit(ctx->identifier()).as>()); + if (ctx->nestedIdentifier()) return std::make_shared(visit(ctx->nestedIdentifier()).as>()); __builtin_unreachable(); } diff --git a/src/Parsers/New/ClickHouseParser.cpp b/src/Parsers/New/ClickHouseParser.cpp index 1be89f63a86..174f838f19d 100644 --- a/src/Parsers/New/ClickHouseParser.cpp +++ b/src/Parsers/New/ClickHouseParser.cpp @@ -16430,8 +16430,8 @@ ClickHouseParser::TableArgExprContext::TableArgExprContext(ParserRuleContext *pa : ParserRuleContext(parent, invokingState) { } -ClickHouseParser::IdentifierContext* ClickHouseParser::TableArgExprContext::identifier() { - return getRuleContext(0); +ClickHouseParser::NestedIdentifierContext* ClickHouseParser::TableArgExprContext::nestedIdentifier() { + return getRuleContext(0); } ClickHouseParser::TableFunctionExprContext* ClickHouseParser::TableArgExprContext::tableFunctionExpr() { @@ -16468,7 +16468,7 @@ ClickHouseParser::TableArgExprContext* ClickHouseParser::tableArgExpr() { case 1: { enterOuterAlt(_localctx, 1); setState(1859); - identifier(); + nestedIdentifier(); break; } @@ -20142,7 +20142,7 @@ ClickHouseParser::Initializer::Initializer() { 0x2, 0x73f, 0x741, 0x5, 0xc4, 0x63, 0x2, 0x740, 0x73e, 0x3, 0x2, 0x2, 0x2, 0x741, 0x744, 0x3, 0x2, 0x2, 0x2, 0x742, 0x740, 0x3, 0x2, 0x2, 0x2, 0x742, 0x743, 0x3, 0x2, 0x2, 0x2, 0x743, 0xc3, 0x3, 0x2, 0x2, 0x2, - 0x744, 0x742, 0x3, 0x2, 0x2, 0x2, 0x745, 0x749, 0x5, 0xd6, 0x6c, 0x2, + 0x744, 0x742, 0x3, 0x2, 0x2, 0x2, 0x745, 0x749, 0x5, 0xba, 0x5e, 0x2, 0x746, 0x749, 0x5, 0xbe, 0x60, 0x2, 0x747, 0x749, 0x5, 0xcc, 0x67, 0x2, 0x748, 0x745, 0x3, 0x2, 0x2, 0x2, 0x748, 0x746, 0x3, 0x2, 0x2, 0x2, 0x748, 0x747, 0x3, 0x2, 0x2, 0x2, 0x749, 0xc5, 0x3, 0x2, 0x2, 0x2, 0x74a, diff --git a/src/Parsers/New/ClickHouseParser.g4 b/src/Parsers/New/ClickHouseParser.g4 index 5cab874e329..28e5b1217ab 100644 --- a/src/Parsers/New/ClickHouseParser.g4 +++ b/src/Parsers/New/ClickHouseParser.g4 @@ -428,7 +428,7 @@ tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; tableIdentifier: (databaseIdentifier DOT)? identifier; tableArgList: tableArgExpr (COMMA tableArgExpr)*; tableArgExpr - : identifier + : nestedIdentifier | tableFunctionExpr | literal ; diff --git a/src/Parsers/New/ClickHouseParser.h b/src/Parsers/New/ClickHouseParser.h index 8754559dcc2..c860932ba1c 100644 --- a/src/Parsers/New/ClickHouseParser.h +++ b/src/Parsers/New/ClickHouseParser.h @@ -3064,7 +3064,7 @@ public: public: TableArgExprContext(antlr4::ParserRuleContext *parent, size_t invokingState); virtual size_t getRuleIndex() const override; - IdentifierContext *identifier(); + NestedIdentifierContext *nestedIdentifier(); TableFunctionExprContext *tableFunctionExpr(); LiteralContext *literal(); From 26c59961ae74a2879c3425c9f361002b10b927a9 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 12 Jun 2021 12:27:05 +0300 Subject: [PATCH 107/491] Minor style changes in StorageMaterializedView::read, StorageMerge::createSources --- src/Storages/StorageMaterializedView.cpp | 27 ++++++++++++------------ src/Storages/StorageMerge.cpp | 18 ++++------------ 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 733cbaed8bb..927a3c232ab 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -47,6 +47,17 @@ static inline String generateInnerTableName(const StorageID & view_id) return ".inner." + view_id.getTableName(); } +/// Remove columns from target_header that does not exists in src_header +static void removeNonCommonColumns(const Block & src_header, Block & target_header) +{ + std::set target_only_positions; + for (const auto & column : target_header) + { + if (!src_header.has(column.name)) + target_only_positions.insert(target_header.getPositionByName(column.name)); + } + target_header.erase(target_only_positions); +} StorageMaterializedView::StorageMaterializedView( const StorageID & table_id_, @@ -168,26 +179,14 @@ void StorageMaterializedView::read( auto target_header = query_plan.getCurrentDataStream().header; /// No need to convert columns that does not exists in MV - std::set target_only_positions; - for (const auto & column : target_header) - { - if (!mv_header.has(column.name)) - target_only_positions.insert(target_header.getPositionByName(column.name)); - } - target_header.erase(target_only_positions); + removeNonCommonColumns(mv_header, target_header); /// No need to convert columns that does not exists in the result header. /// /// Distributed storage may process query up to the specific stage, and /// so the result header may not include all the columns from the /// materialized view. - std::set source_only_positions; - for (const auto & column : mv_header) - { - if (!target_header.has(column.name)) - source_only_positions.insert(mv_header.getPositionByName(column.name)); - } - mv_header.erase(source_only_positions); + removeNonCommonColumns(target_header, mv_header); if (!blocksHaveEqualStructure(mv_header, target_header)) { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5a8864141c4..965c4f15290 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -41,18 +41,6 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; } -namespace -{ - -TreeRewriterResult modifySelect(ASTSelectQuery & select, const TreeRewriterResult & rewriter_result, ContextPtr context) -{ - TreeRewriterResult new_rewriter_result = rewriter_result; - removeJoin(select, new_rewriter_result, context); - return new_rewriter_result; -} - -} - StorageMerge::StorageMerge( const StorageID & table_id_, const ColumnsDescription & columns_, @@ -147,7 +135,7 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( /// should be done on the initiator always. /// /// Since in case of JOIN query on shards will receive query w/o JOIN (and their columns). - /// (see modifySelect()/removeJoin()) + /// (see removeJoin()) /// /// And for this we need to return FetchColumns. if (const auto * select = query_info.query->as(); select && hasJoin(*select)) @@ -297,7 +285,9 @@ Pipe StorageMerge::createSources( /// Original query could contain JOIN but we need only the first joined table and its columns. auto & modified_select = modified_query_info.query->as(); - auto new_analyzer_res = modifySelect(modified_select, *query_info.syntax_analyzer_result, modified_context); + + TreeRewriterResult new_analyzer_res = *query_info.syntax_analyzer_result; + removeJoin(modified_select, new_analyzer_res, modified_context); modified_query_info.syntax_analyzer_result = std::make_shared(std::move(new_analyzer_res)); VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", table_name); From 0c82d45dea4a4140bdda56bd04796baba1d5c918 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Sat, 12 Jun 2021 14:35:34 +0300 Subject: [PATCH 108/491] Missed initialize in ReadBufferFromS3 --- src/IO/ReadBufferFromS3.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index e492f44cfb5..625177bd8c0 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -47,6 +47,8 @@ bool ReadBufferFromS3::nextImpl() bool next_result = false; auto sleep_time_with_backoff_milliseconds = std::chrono::milliseconds(100); + impl = initialize(); + for (size_t attempt = 0; attempt < max_single_read_retries; ++attempt) { try From 47252e07f3aef1430a50388fa65abcd26c7bf6af Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Sat, 12 Jun 2021 15:57:14 +0300 Subject: [PATCH 109/491] Remove trailing whitespaces --- src/IO/ReadBufferFromS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 625177bd8c0..c04a937cc6a 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -48,7 +48,7 @@ bool ReadBufferFromS3::nextImpl() auto sleep_time_with_backoff_milliseconds = std::chrono::milliseconds(100); impl = initialize(); - + for (size_t attempt = 0; attempt < max_single_read_retries; ++attempt) { try From 76bb1a569f1d64e838ba4b97d77df1a39dda6e72 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Sat, 12 Jun 2021 20:46:26 +0300 Subject: [PATCH 110/491] Description for another interval record --- docs/en/sql-reference/operators/index.md | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 31fce7f72b3..8afb77cff8b 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -188,6 +188,45 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV └─────────────────────┴────────────────────────────────────────────────────────────┘ ``` +You can set the period without using `INTERVAL` by multiplying seconds, minutes, and hours. For example, a period of one day can be set at `60*60*24`. + +!!! note "Note" + Syntax `now() + 60*60*24` doesn't consider time settings. For example, daylight saving time. + +The `INTERVAL` syntax or `addDays`-function is always preferred. + +Examples: + +``` sql +SELECT now() AS current_date_time, current_date_time + 60*60*96; +``` + +``` text +┌───current_date_time─┬─plus(now(), multiply(multiply(60, 60), 96))─┐ +│ 2021-06-12 17:34:49 │ 2021-06-16 17:34:49 │ +└─────────────────────┴─────────────────────────────────────────────┘ +``` + +``` sql +SELECT now() AS current_date_time, current_date_time + 60*60*24 + 60*60*2; +``` + +``` text +┌───current_date_time─┬─plus(plus(now(), multiply(multiply(60, 60), 24)), multiply(multiply(60, 60), 2))─┐ +│ 2021-06-12 17:36:30 │ 2021-06-13 19:36:30 │ +└─────────────────────┴──────────────────────────────────────────────────────────────────────────────────┘ +``` + +``` sql +SELECT now() AS current_date_time, current_date_time + 60*23; +``` + +``` text +┌───current_date_time─┬─plus(now(), multiply(60, 23))─┐ +│ 2021-06-12 17:38:02 │ 2021-06-12 18:01:02 │ +└─────────────────────┴───────────────────────────────┘ +``` + **See Also** - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type From 6ce0c1ef3c8f2ce39581b2edeb5bb04e820316df Mon Sep 17 00:00:00 2001 From: meoww-bot <14239840+meoww-bot@users.noreply.github.com> Date: Sun, 13 Jun 2021 11:16:34 +0800 Subject: [PATCH 111/491] Create zh translation for postgresql.md --- .../table-engines/integrations/postgresql.md | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 docs/zh/engines/table-engines/integrations/postgresql.md diff --git a/docs/zh/engines/table-engines/integrations/postgresql.md b/docs/zh/engines/table-engines/integrations/postgresql.md new file mode 100644 index 00000000000..9baebd14aff --- /dev/null +++ b/docs/zh/engines/table-engines/integrations/postgresql.md @@ -0,0 +1,145 @@ +--- +toc_priority: 11 +toc_title: PostgreSQL +--- + +# PostgreSQL {#postgresql} + +PostgreSQL 引擎允许对存储在远程 PostgreSQL 服务器上的数据进行 `SELECT` 和 `INSERT` 查询. + +## 创建一张表 {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... +) ENGINE = PostgreSQL('host:port', 'database', 'table', 'user', 'password'[, `schema`]); +``` + +详情请见 [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) 查询. + +表结构可以与 PostgreSQL 源表结构不同: + +- 列名应与 PostgreSQL 源表中的列名相同，但您可以按任何顺序使用其中的一些列。 +- 列类型可能与源表中的列类型不同。 ClickHouse尝试将数值[映射](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) 到ClickHouse的数据类型。 +- 设置 `external_table_functions_use_nulls` 来定义如何处理 Nullable 列. 默认值是 1, 当设置为 0 时 - 表函数将不会使用 nullable 列，而是插入默认值来代替 null. 这同样适用于数组数据类型中的 null 值. + +**引擎参数** + +- `host:port` — PostgreSQL 服务器地址. +- `database` — 数据库名称. +- `table` — 表名称. +- `user` — PostgreSQL 用户. +- `password` — 用户密码. +- `schema` — Non-default table schema. 可选. + +## 实施细节 {#implementation-details} + +在 PostgreSQL 上的 `SELECT` 查询以 `COPY (SELECT ...) TO STDOUT` 的方式在只读 PostgreSQL 事务中运行，每次 `SELECT` 查询后提交。 + +简单的 `WHERE` 子句，如`=`，`！=`，`>`，`>=`，`<`，`<=`，和`IN`是在PostgreSQL 服务器上执行。 + +所有的连接、聚合、排序、`IN [ array ]`条件和`LIMIT`采样约束都是在 PostgreSQL 的查询结束后才在ClickHouse中执行的。 + +在 PostgreSQL 上的 `INSERT` 查询以 `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` 的方式在 PostgreSQL 事务中运行，每条 `INSERT` 语句后自动提交。 + +PostgreSQL 的 `Array` 类型会被转换为 ClickHouse 数组。 + +!!! info "Note" + 要小心 - 一个在 PostgreSQL 中的数组数据，像`type_name[]`这样创建，可以在同一列的不同表行中包含不同维度的多维数组。但是在 ClickHouse 中，只允许在同一列的所有表行中包含相同维数的多维数组。 + +支持设置 PostgreSQL 字典源中 Replicas 的优先级。地图中的数字越大，优先级就越低。最高的优先级是 `0`。 + +在下面的例子中，副本`example01-1`有最高的优先级。 + +```xml + + 5432 + clickhouse + qwerty + + example01-1 + 1 + + + example01-2 + 2 + + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +## 用法示例 {#usage-example} + +PostgreSQL 中的表: + +``` text +postgres=# CREATE TABLE "public"."test" ( +"int_id" SERIAL, +"int_nullable" INT NULL DEFAULT NULL, +"float" FLOAT NOT NULL, +"str" VARCHAR(100) NOT NULL DEFAULT '', +"float_nullable" FLOAT NULL DEFAULT NULL, +PRIMARY KEY (int_id)); + +CREATE TABLE + +postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2); +INSERT 0 1 + +postgresql> SELECT * FROM test; + int_id | int_nullable | float | str | float_nullable + --------+--------------+-------+------+---------------- + 1 | | 2 | test | + (1 row) +``` + +ClickHouse 中的表, 从上面创建的 PostgreSQL 表中检索数据: + +``` sql +CREATE TABLE default.postgresql_table +( + `float_nullable` Nullable(Float32), + `str` String, + `int_id` Int32 +) +ENGINE = PostgreSQL('localhost:5432', 'public', 'test', 'postges_user', 'postgres_password'); +``` + +``` sql +SELECT * FROM postgresql_table WHERE str IN ('test'); +``` + +``` text +┌─float_nullable─┬─str──┬─int_id─┐ +│ ᴺᵁᴸᴸ │ test │ 1 │ +└────────────────┴──────┴────────┘ +``` + +使用非默认的模式: + +```text +postgres=# CREATE SCHEMA "nice.schema"; + +postgres=# CREATE TABLE "nice.schema"."nice.table" (a integer); + +postgres=# INSERT INTO "nice.schema"."nice.table" SELECT i FROM generate_series(0, 99) as t(i) +``` + +```sql +CREATE TABLE pg_table_schema_with_dots (a UInt32) + ENGINE PostgreSQL('localhost:5432', 'clickhouse', 'nice.table', 'postgrsql_user', 'password', 'nice.schema'); +``` + +**另请参阅** + +- [`postgresql` 表函数](../../../sql-reference/table-functions/postgresql.md) +- [使用 PostgreSQL 作为外部字典的来源](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) + +[原始文章](https://clickhouse.tech/docs/en/engines/table-engines/integrations/postgresql/) From 6d2c062859de06a727ea320777b74d03345e20dc Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sun, 13 Jun 2021 15:20:48 +0300 Subject: [PATCH 112/491] edited EN --- .../parametric-functions.md | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index e82cb4882a0..90b469746a5 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -509,7 +509,7 @@ Same behavior as [sumMap](../../sql-reference/aggregate-functions/reference/summ ## sequenceNextNode {#sequenceNextNode} -Returns a value of next event that matched an event chain. +Returns a value of the next event that matched an event chain. _Experimental function, `SET allow_experimental_funnel_functions = 1` to enable it._ @@ -520,33 +520,36 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event ``` **Parameters** -- `direction` - Used to navigate to directions. - - forward : Moving forward - - backward: Moving backward -- `base` - Used to set the base point. - - head : Set the base point to the first event - - tail : Set the base point to the last event - - first_match : Set the base point to the first matched event1 - - last_match : Set the base point to the last matched event1 +- `direction` — Used to navigate to directions. + - forward — Moving forward. + - backward — Moving backward. + +- `base` — Used to set the base point. + - head — Set the base point to the first event. + - tail — Set the base point to the last event. + - first_match — Set the base point to the first matched event1. + - last_match — Set the base point to the last matched event1. **Arguments** -- `timestamp` — Name of the column containing the timestamp. Data types supported: `Date`, `DateTime` and other unsigned integer types. -- `event_column` — Name of the column containing the value of the next event to be returned. Data types supported: `String` and `Nullable(String)` + +- `timestamp` — Name of the column containing the timestamp. Data types supported: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) and other unsigned integer types. +- `event_column` — Name of the column containing the value of the next event to be returned. Data types supported: [String](../../sql-reference/data-types/string.md) and [Nullable(String)](../../sql-reference/data-types/nullable.md). - `base_condition` — Condition that the base point must fulfill. -- `cond` — Conditions describing the chain of events. `UInt8` +- `cond` — Conditions describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). -**Returned value** -- `event_column[next_index]` - if the pattern is matched and next value exists. -- `NULL` - if the pattern isn’t matched or next value doesn't exist. +**Returned values** -Type: `Nullable(String)`. +- `event_column[next_index]` — If the pattern is matched and next value exists. +- `NULL` - If the pattern isn’t matched or next value doesn't exist. + +Type: [Nullable(String)](../../sql-reference/data-types/nullable.md). **Example** It can be used when events are A->B->C->E->F and you want to know the event following B->C, which is E. -The query statement searching the event following A->B : +The query statement searching the event following A->B: ``` sql CREATE TABLE test_flow ( @@ -572,7 +575,7 @@ Result: **Behavior for `forward` and `head`** -```SQL +``` sql ALTER TABLE test_flow DELETE WHERE 1 = 1 settings mutations_sync = 1; INSERT INTO test_flow VALUES (1, 1, 'Home') (2, 1, 'Gift') (3, 1, 'Exit'); @@ -580,7 +583,7 @@ INSERT INTO test_flow VALUES (1, 2, 'Home') (2, 2, 'Home') (3, 2, 'Gift') (4, 2, INSERT INTO test_flow VALUES (1, 3, 'Gift') (2, 3, 'Home') (3, 3, 'Gift') (4, 3, 'Basket'); ``` -```SQL +``` sql SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'Home', page = 'Home', page = 'Gift') FROM test_flow GROUP BY id; dt id page @@ -601,7 +604,7 @@ SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'Home', page = ' **Behavior for `backward` and `tail`** -```SQL +``` sql SELECT id, sequenceNextNode('backward', 'tail')(dt, page, page = 'Basket', page = 'Basket', page = 'Gift') FROM test_flow GROUP BY id; dt id page @@ -623,7 +626,7 @@ SELECT id, sequenceNextNode('backward', 'tail')(dt, page, page = 'Basket', page **Behavior for `forward` and `first_match`** -```SQL +``` sql SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; dt id page @@ -642,7 +645,7 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p 1970-01-01 09:00:04 3 Basket ``` -```SQL +``` sql SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; dt id page @@ -664,7 +667,7 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p **Behavior for `backward` and `last_match`** -```SQL +``` sql SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; dt id page @@ -683,7 +686,7 @@ SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', p 1970-01-01 09:00:04 3 Basket ``` -```SQL +``` sql SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; dt id page @@ -705,7 +708,7 @@ SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', p **Behavior for `base_condition`** -```SQL +``` sql CREATE TABLE test_flow_basecond ( `dt` DateTime, @@ -720,7 +723,7 @@ ORDER BY id INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3, 1, 'B', 'ref2') (4, 1, 'B', 'ref1'); ``` -```SQL +``` sql SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref @@ -730,7 +733,7 @@ SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A 1970-01-01 09:00:04 1 B ref1 ``` -```SQL +``` sql SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = 'B') FROM test_flow_basecond GROUP BY id; dt id page ref @@ -740,7 +743,7 @@ SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = ' 1970-01-01 09:00:04 1 B ref1 // The tail can't be base point becasue the ref column of the tail unmatched with 'ref4'. ``` -```SQL +``` sql SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref @@ -750,7 +753,7 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', pa 1970-01-01 09:00:04 1 B ref1 ``` -```SQL +``` sql SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, ref = 'ref2', page = 'B') FROM test_flow_basecond GROUP BY id; dt id page ref From 8ddb7f7fbfdc07ae70cd29ef1291fe6027f3571d Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sun, 13 Jun 2021 15:51:26 +0300 Subject: [PATCH 113/491] more edits in EN --- .../aggregate-functions/parametric-functions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 90b469746a5..e0953f5fa28 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -718,7 +718,7 @@ CREATE TABLE test_flow_basecond ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(dt) -ORDER BY id +ORDER BY id; INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3, 1, 'B', 'ref2') (4, 1, 'B', 'ref1'); ``` @@ -727,7 +727,7 @@ INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3 SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref - 1970-01-01 09:00:01 1 A ref4 // The head can't be base point becasue the ref column of the head unmatched with 'ref1'. + 1970-01-01 09:00:01 1 A ref4 // The head can not be base point because the ref column of the head unmatched with 'ref1'. 1970-01-01 09:00:02 1 A ref3 1970-01-01 09:00:03 1 B ref2 1970-01-01 09:00:04 1 B ref1 @@ -740,14 +740,14 @@ SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = ' 1970-01-01 09:00:01 1 A ref4 1970-01-01 09:00:02 1 A ref3 1970-01-01 09:00:03 1 B ref2 - 1970-01-01 09:00:04 1 B ref1 // The tail can't be base point becasue the ref column of the tail unmatched with 'ref4'. + 1970-01-01 09:00:04 1 B ref1 // The tail can not be base point because the ref column of the tail unmatched with 'ref4'. ``` ``` sql SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref - 1970-01-01 09:00:01 1 A ref4 // This row can't be base point becasue the ref column unmatched with 'ref3'. + 1970-01-01 09:00:01 1 A ref4 // This row can not be base point because the ref column unmatched with 'ref3'. 1970-01-01 09:00:02 1 A ref3 // Base point 1970-01-01 09:00:03 1 B ref2 // The result 1970-01-01 09:00:04 1 B ref1 @@ -760,5 +760,5 @@ SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, ref = 'ref2', pa 1970-01-01 09:00:01 1 A ref4 1970-01-01 09:00:02 1 A ref3 // The result 1970-01-01 09:00:03 1 B ref2 // Base point - 1970-01-01 09:00:04 1 B ref1 // This row can't be base point becasue the ref column unmatched with 'ref2'. + 1970-01-01 09:00:04 1 B ref1 // This row can not be base point because the ref column unmatched with 'ref2'. ``` From 8e763d7fb2888e4d5f8983902bfca85704cc6ed6 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sun, 13 Jun 2021 16:25:00 +0300 Subject: [PATCH 114/491] added RU --- .../parametric-functions.md | 255 ++++++++++++++++++ 1 file changed, 255 insertions(+) diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 508c8de2a58..7377ce1f71d 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -496,3 +496,258 @@ FROM Решение: пишем в запросе GROUP BY SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 ``` +## sequenceNextNode {#sequenceNextNode} + +Возвращает значение следующего события, соответствующего цепочке событий. + +_Экспериментальная функция, чтобы включить ее, выполните: `SET allow_experimental_funnel_functions = 1`._ + +**Синтаксис** + +``` sql +sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event1, event2, event3, ...) +``` + +**Параметры** + +- `direction` — используется для навигации по направлениям. + - forward — двигаться вперед. + - backward — двигаться назад. + +- `base` — используется для задания начальной точки. + - head — установить начальную точку на первое событие цепочки. + - tail — установить начальную точку на последнее событие цепочки. + - first_match — установить начальную точку на первое соответствующее событие1. + - last_match — установить начальную точку на последнее соответствующее событие1. + +**Аргументы** + +- `timestamp` — название столбца, содержащего `timestamp`. Поддерживаемые типы данных: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) and other unsigned integer types. +- `event_column` — название столбца, содержащего значение следующего возвращаемого события. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) and [Nullable(String)](../../sql-reference/data-types/nullable.md). +- `base_condition` — условие, которому должна соответствовать исходная точка. +- `cond` — условия, описывающие цепочку событий. [UInt8](../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- `event_column[next_index]` — если шаблон совпал и существует следующее значение. +- `NULL` - если шаблон не совпал или следующее значение не существует. + +Тип: [Nullable(String)](../../sql-reference/data-types/nullable.md). + +**Пример** + +Функцию можно использовать, если есть цепочка событий A-> B-> C-> E-> F, и вы хотите определить событие, следующее за B-> C, то есть E. + +Оператор запроса ищет событие после A-> B: + +``` sql +CREATE TABLE test_flow ( + dt DateTime, + id int, + page String) +ENGINE = MergeTree() +PARTITION BY toYYYYMMDD(dt) +ORDER BY id; + +INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'E') (5, 1, 'F'); + +SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'A', page = 'A', page = 'B') as next_flow FROM test_flow GROUP BY id; +``` + +Результат: + +``` text +┌─id─┬─next_flow─┐ +│ 1 │ C │ +└────┴───────────┘ +``` + +**Поведение для `forward` и `head`** + +``` sql +ALTER TABLE test_flow DELETE WHERE 1 = 1 settings mutations_sync = 1; + +INSERT INTO test_flow VALUES (1, 1, 'Home') (2, 1, 'Gift') (3, 1, 'Exit'); +INSERT INTO test_flow VALUES (1, 2, 'Home') (2, 2, 'Home') (3, 2, 'Gift') (4, 2, 'Basket'); +INSERT INTO test_flow VALUES (1, 3, 'Gift') (2, 3, 'Home') (3, 3, 'Gift') (4, 3, 'Basket'); +``` + +``` sql +SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'Home', page = 'Home', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page + 1970-01-01 09:00:01 1 Home // Base point, Matched with Home + 1970-01-01 09:00:02 1 Gift // Matched with Gift + 1970-01-01 09:00:03 1 Exit // The result + + 1970-01-01 09:00:01 2 Home // Base point, Matched with Home + 1970-01-01 09:00:02 2 Home // Unmatched with Gift + 1970-01-01 09:00:03 2 Gift + 1970-01-01 09:00:04 2 Basket + + 1970-01-01 09:00:01 3 Gift // Base point, Unmatched with Home + 1970-01-01 09:00:02 3 Home + 1970-01-01 09:00:03 3 Gift + 1970-01-01 09:00:04 3 Basket +``` + +**Поведение для `backward` и `tail`** + +``` sql +SELECT id, sequenceNextNode('backward', 'tail')(dt, page, page = 'Basket', page = 'Basket', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift +1970-01-01 09:00:03 1 Exit // Base point, Unmatched with Basket + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home // The result +1970-01-01 09:00:03 2 Gift // Matched with Gift +1970-01-01 09:00:04 2 Basket // Base point, Matched with Basket + +1970-01-01 09:00:01 3 Gift +1970-01-01 09:00:02 3 Home // The result +1970-01-01 09:00:03 3 Gift // Base point, Matched with Gift +1970-01-01 09:00:04 3 Basket // Base point, Matched with Basket +``` + + +**Поведение для `forward` и `first_match`** + +``` sql +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit // The result + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket The result + +1970-01-01 09:00:01 3 Gift // Base point +1970-01-01 09:00:02 3 Home // Thre result +1970-01-01 09:00:03 3 Gift +1970-01-01 09:00:04 3 Basket +``` + +``` sql +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit // Unmatched with Home + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket // Unmatched with Home + +1970-01-01 09:00:01 3 Gift // Base point +1970-01-01 09:00:02 3 Home // Matched with Home +1970-01-01 09:00:03 3 Gift // The result +1970-01-01 09:00:04 3 Basket +``` + + +**Поведение для `backward` и `last_match`** + +``` sql +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home // The result +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home // The result +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket + +1970-01-01 09:00:01 3 Gift +1970-01-01 09:00:02 3 Home // The result +1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:04 3 Basket +``` + +``` sql +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home // Matched with Home, the result is null +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit + +1970-01-01 09:00:01 2 Home // The result +1970-01-01 09:00:02 2 Home // Matched with Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket + +1970-01-01 09:00:01 3 Gift // The result +1970-01-01 09:00:02 3 Home // Matched with Home +1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:04 3 Basket +``` + + +**Поведение для `base_condition`** + +``` sql +CREATE TABLE test_flow_basecond +( + `dt` DateTime, + `id` int, + `page` String, + `ref` String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(dt) +ORDER BY id; + +INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3, 1, 'B', 'ref2') (4, 1, 'B', 'ref1'); +``` + +``` sql +SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 // The head can not be base point because the ref column of the head unmatched with 'ref1'. + 1970-01-01 09:00:02 1 A ref3 + 1970-01-01 09:00:03 1 B ref2 + 1970-01-01 09:00:04 1 B ref1 + ``` + +``` sql +SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = 'B') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 + 1970-01-01 09:00:02 1 A ref3 + 1970-01-01 09:00:03 1 B ref2 + 1970-01-01 09:00:04 1 B ref1 // The tail can not be base point because the ref column of the tail unmatched with 'ref4'. +``` + +``` sql +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', page = 'A') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 // This row can not be base point because the ref column unmatched with 'ref3'. + 1970-01-01 09:00:02 1 A ref3 // Base point + 1970-01-01 09:00:03 1 B ref2 // The result + 1970-01-01 09:00:04 1 B ref1 +``` + +``` sql +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, ref = 'ref2', page = 'B') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 + 1970-01-01 09:00:02 1 A ref3 // The result + 1970-01-01 09:00:03 1 B ref2 // Base point + 1970-01-01 09:00:04 1 B ref1 // This row can not be base point because the ref column unmatched with 'ref2'. +``` From dc5052a79fb6a05ca69e9252455f7a3d7adeab3f Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sun, 13 Jun 2021 16:43:23 +0300 Subject: [PATCH 115/491] edited RU --- .../aggregate-functions/parametric-functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 7377ce1f71d..bb0e951a2f1 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -522,15 +522,15 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event **Аргументы** -- `timestamp` — название столбца, содержащего `timestamp`. Поддерживаемые типы данных: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) and other unsigned integer types. -- `event_column` — название столбца, содержащего значение следующего возвращаемого события. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) and [Nullable(String)](../../sql-reference/data-types/nullable.md). +- `timestamp` — название столбца, содержащего `timestamp`. Поддерживаемые типы данных: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) и другие беззнаковые целые типы. +- `event_column` — название столбца, содержащего значение следующего возвращаемого события. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [Nullable(String)](../../sql-reference/data-types/nullable.md). - `base_condition` — условие, которому должна соответствовать исходная точка. - `cond` — условия, описывающие цепочку событий. [UInt8](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** - `event_column[next_index]` — если шаблон совпал и существует следующее значение. -- `NULL` - если шаблон не совпал или следующее значение не существует. +- `NULL` — если шаблон не совпал или следующее значение не существует. Тип: [Nullable(String)](../../sql-reference/data-types/nullable.md). From d5bd761db98b17b6f4a43db08b3f48a522ba497f Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Sun, 13 Jun 2021 22:27:54 +0300 Subject: [PATCH 116/491] Typo Co-authored-by: Nikita Mikhaylov --- src/Parsers/ASTIdentifier.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 7df31244f35..5fc446ae477 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -48,7 +48,7 @@ public: const String & name() const; void restoreTable(); // TODO(ilezhankin): get rid of this - std::shared_ptr createTable() const; // return |nullptr| if identifier is not table. + std::shared_ptr createTable() const; // returns |nullptr| if identifier is not table. protected: String full_name; From b40f83cd21cb9ffe52c987345b2d5c1306f7561a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 13 Jun 2021 22:59:17 +0300 Subject: [PATCH 117/491] Fixed the annoying Arcadia, it is bogging my weekend. --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index dac43ff5d4b..f15a92860de 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -241,3 +241,4 @@ 01882_scalar_subquery_exception 01882_check_max_parts_to_merge_at_once 01892_setting_limit_offset_distributed +01901_test_attach_partition_from From 79b21caa287b4b7065017094c636d31d0e4da43f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 13 Jun 2021 23:52:35 +0300 Subject: [PATCH 118/491] Add test for #4113 --- .../0_stateless/01907_multiple_aliases.reference | 1 + tests/queries/0_stateless/01907_multiple_aliases.sql | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/01907_multiple_aliases.reference create mode 100644 tests/queries/0_stateless/01907_multiple_aliases.sql diff --git a/tests/queries/0_stateless/01907_multiple_aliases.reference b/tests/queries/0_stateless/01907_multiple_aliases.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01907_multiple_aliases.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01907_multiple_aliases.sql b/tests/queries/0_stateless/01907_multiple_aliases.sql new file mode 100644 index 00000000000..611960a5205 --- /dev/null +++ b/tests/queries/0_stateless/01907_multiple_aliases.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (d Date, z UInt32) ENGINE = MergeTree(d, (z), 1); + +INSERT INTO t VALUES ('2017-01-01', 1); + +WITH (d < '2018-01-01') AND (d < '2018-01-02') AS x +SELECT 1 +FROM t +WHERE x; + +DROP TABLE t; From 82502a8d7e3cd7e7a885e601fe91bd00d16bb997 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 00:07:37 +0300 Subject: [PATCH 119/491] Add test for #11535 --- .../01908_with_unknown_column.reference | 3 ++ .../0_stateless/01908_with_unknown_column.sql | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/queries/0_stateless/01908_with_unknown_column.reference create mode 100644 tests/queries/0_stateless/01908_with_unknown_column.sql diff --git a/tests/queries/0_stateless/01908_with_unknown_column.reference b/tests/queries/0_stateless/01908_with_unknown_column.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/tests/queries/0_stateless/01908_with_unknown_column.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/01908_with_unknown_column.sql b/tests/queries/0_stateless/01908_with_unknown_column.sql new file mode 100644 index 00000000000..c3bce12d41e --- /dev/null +++ b/tests/queries/0_stateless/01908_with_unknown_column.sql @@ -0,0 +1,30 @@ +select a +from ( + with a+1 as aa, + sumIf(aa, b > 0) as aaif + select a, aaif + FROM (select 1 as a, 2 as b) + GROUP BY a +) as V; + +select a +from ( + with a+1 as aa + -- , sumIf(c, b > 0) as aaif + , sum(if(b>0,c,0)) as aaif2 + select a, aaif2 + FROM + (select 1 as a, 2 as b, 3 as c) + GROUP BY a +) as V; + +select a +from ( + with a+1 as aa + -- , sumIf(c, b > 0) as aaif + -- , sum(if(b>0,c,0)) as aaif2 + select a, sumIf(c, b > 0) as aaif3 + FROM + (select 1 as a, 2 as b, 3 as c) + GROUP BY a +) as V; From 01eaaace8b61c15e5f1c3a5aca0f839aa3f97cfe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 02:50:34 +0300 Subject: [PATCH 120/491] Add a test for #9932 --- .../01910_view_dictionary.reference | 6 +++ .../0_stateless/01910_view_dictionary.sql | 45 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/queries/0_stateless/01910_view_dictionary.reference create mode 100644 tests/queries/0_stateless/01910_view_dictionary.sql diff --git a/tests/queries/0_stateless/01910_view_dictionary.reference b/tests/queries/0_stateless/01910_view_dictionary.reference new file mode 100644 index 00000000000..e40484d76c4 --- /dev/null +++ b/tests/queries/0_stateless/01910_view_dictionary.reference @@ -0,0 +1,6 @@ +1 One Один +2 Two Два +3 Three Три +One Один +Two Два +Three Три diff --git a/tests/queries/0_stateless/01910_view_dictionary.sql b/tests/queries/0_stateless/01910_view_dictionary.sql new file mode 100644 index 00000000000..28c50daeba1 --- /dev/null +++ b/tests/queries/0_stateless/01910_view_dictionary.sql @@ -0,0 +1,45 @@ +DROP TABLE IF EXISTS dictionary_source_en; +DROP TABLE IF EXISTS dictionary_source_ru; +DROP TABLE IF EXISTS dictionary_source_view; +DROP DICTIONARY IF EXISTS flat_dictionary; + +CREATE TABLE dictionary_source_en +( + id UInt64, + value String +) ENGINE = TinyLog; + +INSERT INTO dictionary_source_en VALUES (1, 'One'), (2,'Two'), (3, 'Three'); + +CREATE TABLE dictionary_source_ru +( + id UInt64, + value String +) ENGINE = TinyLog; + +INSERT INTO dictionary_source_ru VALUES (1, 'Один'), (2,'Два'), (3, 'Три'); + +CREATE VIEW dictionary_source_view AS SELECT id, dictionary_source_en.value as value_en, dictionary_source_ru.value as value_ru FROM dictionary_source_en LEFT JOIN dictionary_source_ru USING (id); + +select * from dictionary_source_view; + +CREATE DICTIONARY flat_dictionary +( + id UInt64, + value_en String, + value_ru String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' PASSWORD '' TABLE 'dictionary_source_view')) +LIFETIME(MIN 1 MAX 1000) +LAYOUT(FLAT()); + +SELECT + dictGet(concat(currentDatabase(), '.flat_dictionary'), 'value_en', number + 1), + dictGet(concat(currentDatabase(), '.flat_dictionary'), 'value_ru', number + 1) +FROM numbers(3); + +DROP TABLE dictionary_source_en; +DROP TABLE dictionary_source_ru; +DROP TABLE dictionary_source_view; +DROP DICTIONARY flat_dictionary; From dbff1edcd34ff05ecf657b50f3164e06ded73a81 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 05:26:05 +0300 Subject: [PATCH 121/491] Fix memory tracking of aggregate function topK --- src/AggregateFunctions/AggregateFunctionTopK.h | 3 ++- src/Common/SpaceSaving.h | 5 +++-- .../queries/0_stateless/01910_memory_tracking_topk.reference | 0 tests/queries/0_stateless/01910_memory_tracking_topk.sql | 4 ++++ 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01910_memory_tracking_topk.reference create mode 100644 tests/queries/0_stateless/01910_memory_tracking_topk.sql diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h index 30d69b8ca7b..1988b1f8b2a 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/src/AggregateFunctions/AggregateFunctionTopK.h @@ -118,7 +118,8 @@ struct AggregateFunctionTopKGenericData * For such columns topK() can be implemented more efficiently (especially for small numeric arrays). */ template -class AggregateFunctionTopKGeneric : public IAggregateFunctionDataHelper> +class AggregateFunctionTopKGeneric + : public IAggregateFunctionDataHelper> { private: using State = AggregateFunctionTopKGenericData; diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h index b7353d803b7..d1e6d079d17 100644 --- a/src/Common/SpaceSaving.h +++ b/src/Common/SpaceSaving.h @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -382,8 +383,8 @@ private: using CounterMap = HashMapWithStackMemory; CounterMap counter_map; - std::vector counter_list; - std::vector alpha_map; + std::vector> counter_list; + std::vector> alpha_map; SpaceSavingArena arena; size_t m_capacity; size_t removed_keys = 0; diff --git a/tests/queries/0_stateless/01910_memory_tracking_topk.reference b/tests/queries/0_stateless/01910_memory_tracking_topk.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01910_memory_tracking_topk.sql b/tests/queries/0_stateless/01910_memory_tracking_topk.sql new file mode 100644 index 00000000000..e6309c1eeb8 --- /dev/null +++ b/tests/queries/0_stateless/01910_memory_tracking_topk.sql @@ -0,0 +1,4 @@ +-- Memory limit must correctly apply, triggering an exception: + +SET max_memory_usage = '100M'; +SELECT length(topK(5592405)(tuple(number))) FROM numbers(10) GROUP BY number; -- { serverError 241 } From 2211ba53d43f0ef76d0fea5da95d5bc425495f91 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 05:48:10 +0300 Subject: [PATCH 122/491] Copy paste some code --- src/Storages/MergeTree/MergeTreePartition.cpp | 129 +++++++++++++++++- 1 file changed, 125 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 4bc3b2c8ba6..894120e6179 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -27,12 +27,133 @@ namespace /// It worked this way until 21.5, and we cannot change it, /// or partition ID will be different in case UUID is used in partition key. /// (It is not recommended to use UUID as partition key). - class LegacyFieldVisitorHash : public FieldVisitorHash + /// NOTE: The code is intentionally copy-pasted, + /// so when FieldVisitorHash is changed, LegacyFieldVisitorHash will not change. + class LegacyFieldVisitorHash : public StaticVisitor<> { + private: + SipHash & hash; public: - using FieldVisitorHash::FieldVisitorHash; - using FieldVisitorHash::operator(); - void operator() (const UUID & x) const { FieldVisitorHash::operator()(x.toUnderType()); } + LegacyFieldVisitorHash(SipHash & hash_) : hash(hash_) {} + + void operator() (const Null &) const + { + UInt8 type = Field::Types::Null; + hash.update(type); + } + void operator() (const UInt64 & x) const + { + UInt8 type = Field::Types::UInt64; + hash.update(type); + hash.update(x); + } + void operator() (const UInt128 & x) const + { + UInt8 type = Field::Types::UInt128; + hash.update(type); + hash.update(x); + } + void operator() (const UInt256 & x) const + { + UInt8 type = Field::Types::UInt256; + hash.update(type); + hash.update(x); + } + void operator() (const Int64 & x) const + { + UInt8 type = Field::Types::Int64; + hash.update(type); + hash.update(x); + } + void operator() (const Int128 & x) const + { + UInt8 type = Field::Types::Int128; + hash.update(type); + hash.update(x); + } + void operator() (const Int256 & x) const + { + UInt8 type = Field::Types::Int256; + hash.update(type); + hash.update(x); + } + void operator() (const UUID & x) const + { + operator()(x.toUnderType()); + } + void operator() (const Float64 & x) const + { + UInt8 type = Field::Types::Float64; + hash.update(type); + hash.update(x); + } + void operator() (const String & x) const + { + UInt8 type = Field::Types::String; + hash.update(type); + hash.update(x.size()); + hash.update(x.data(), x.size()); + } + void operator() (const Array & x) const + { + UInt8 type = Field::Types::Array; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); + } + void operator() (const Tuple & x) const + { + UInt8 type = Field::Types::Tuple; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); + } + void operator() (const Map & x) const + { + UInt8 type = Field::Types::Map; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); + } + void operator() (const DecimalField & x) const + { + UInt8 type = Field::Types::Decimal32; + hash.update(type); + hash.update(x.getValue().value); + } + void operator() (const DecimalField & x) const + { + UInt8 type = Field::Types::Decimal64; + hash.update(type); + hash.update(x.getValue().value); + } + void operator() (const DecimalField & x) const + { + UInt8 type = Field::Types::Decimal128; + hash.update(type); + hash.update(x.getValue().value); + } + void operator() (const DecimalField & x) const + { + UInt8 type = Field::Types::Decimal256; + hash.update(type); + hash.update(x.getValue().value); + } + void operator() (const AggregateFunctionStateData & x) const + { + UInt8 type = Field::Types::AggregateFunctionState; + hash.update(type); + hash.update(x.name.size()); + hash.update(x.name.data(), x.name.size()); + hash.update(x.data.size()); + hash.update(x.data.data(), x.data.size()); + } }; } From cae9b25074dd58655c74ca39ba4af7d40fe66099 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 06:05:27 +0300 Subject: [PATCH 123/491] Minor change --- src/Common/FieldVisitors.h | 6 ------ src/Common/FieldVisitorsAccurateComparison.h | 12 ++++++------ src/Core/Field.h | 6 ++++++ src/Parsers/ASTFunction.cpp | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index ef593dc6ef9..2f7e24e4fad 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -269,12 +269,6 @@ public: void operator() (const AggregateFunctionStateData & x) const; }; -template constexpr bool isDecimalField() { return false; } -template <> constexpr bool isDecimalField>() { return true; } -template <> constexpr bool isDecimalField>() { return true; } -template <> constexpr bool isDecimalField>() { return true; } -template <> constexpr bool isDecimalField>() { return true; } - /** Implements `+=` operation. * Returns false if the result is zero. diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h index cf3cbb208dc..0f605b7da23 100644 --- a/src/Common/FieldVisitorsAccurateComparison.h +++ b/src/Common/FieldVisitorsAccurateComparison.h @@ -37,13 +37,13 @@ public: return accurate::equalsOp(l, r); /// TODO This is wrong (does not respect scale). - if constexpr (isDecimalField() && isDecimalField()) + if constexpr (is_decimal_field && is_decimal_field) return l == r; - if constexpr (isDecimalField() && std::is_arithmetic_v) + if constexpr (is_decimal_field && std::is_arithmetic_v) return l == DecimalField(Decimal256(r), 0); - if constexpr (std::is_arithmetic_v && isDecimalField()) + if constexpr (std::is_arithmetic_v && is_decimal_field) return DecimalField(Decimal256(l), 0) == r; if constexpr (std::is_same_v && std::is_arithmetic_v) @@ -86,13 +86,13 @@ public: return accurate::lessOp(l, r); /// TODO This is wrong (does not respect scale). - if constexpr (isDecimalField() && isDecimalField()) + if constexpr (is_decimal_field && is_decimal_field) return l < r; - if constexpr (isDecimalField() && std::is_arithmetic_v) + if constexpr (is_decimal_field && std::is_arithmetic_v) return l < DecimalField(Decimal256(r), 0); - if constexpr (std::is_arithmetic_v && isDecimalField()) + if constexpr (std::is_arithmetic_v && is_decimal_field) return DecimalField(Decimal256(l), 0) < r; if constexpr (std::is_same_v && std::is_arithmetic_v) diff --git a/src/Core/Field.h b/src/Core/Field.h index 9c970fbbb31..2242e8fddae 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -162,6 +162,12 @@ private: #pragma GCC diagnostic pop #endif +template constexpr bool is_decimal_field = false; +template <> constexpr bool is_decimal_field> = true; +template <> constexpr bool is_decimal_field> = true; +template <> constexpr bool is_decimal_field> = true; +template <> constexpr bool is_decimal_field> = true; + /// char may be signed or unsigned, and behave identically to signed char or unsigned char, /// but they are always three different types. /// signedness of char is different in Linux on x86 and Linux on ARM. diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index cc460f600dd..8df3383f487 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -252,7 +252,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format NO_SANITIZE_UNDEFINED { using ValueType = std::decay_t; - if constexpr (isDecimalField()) + if constexpr (is_decimal_field) { // The parser doesn't create decimal literals, but // they can be produced by constant folding or the From 447d7bb8cdb51e53c825d24cd7f6a4aa468cff55 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 07:13:35 +0300 Subject: [PATCH 124/491] Minor changes --- src/Access/SettingsConstraints.cpp | 2 +- .../AggregateFunctionGroupArrayInsertAt.h | 3 +- .../AggregateFunctionHistogram.cpp | 3 +- .../AggregateFunctionMLMethod.cpp | 2 +- .../AggregateFunctionSumMap.h | 2 +- .../AggregateFunctionTopK.cpp | 2 +- .../AggregateFunctionUniqCombined.cpp | 2 +- .../AggregateFunctionUniqUpTo.cpp | 2 +- src/AggregateFunctions/QuantilesCommon.h | 2 +- src/Columns/ColumnAggregateFunction.cpp | 2 +- src/Common/FieldVisitorConvertToNumber.h | 129 ++++++ src/Common/FieldVisitorDump.cpp | 109 +++++ src/Common/FieldVisitorDump.h | 33 ++ src/Common/FieldVisitorHash.cpp | 149 +++++++ src/Common/FieldVisitorHash.h | 38 ++ src/Common/FieldVisitorSum.cpp | 37 ++ src/Common/FieldVisitorSum.h | 47 ++ src/Common/FieldVisitorToString.cpp | 130 ++++++ src/Common/FieldVisitorToString.h | 33 ++ src/Common/FieldVisitorWriteBinary.cpp | 70 +++ src/Common/FieldVisitorWriteBinary.h | 32 ++ src/Common/FieldVisitors.cpp | 421 ------------------ src/Common/FieldVisitors.h | 278 ------------ src/Common/ya.make | 6 +- src/Core/Block.cpp | 2 +- src/Core/Field.cpp | 16 +- src/Core/Field.h | 8 +- src/Core/MySQL/MySQLReplication.cpp | 3 +- src/Core/SettingsFields.cpp | 2 +- src/Core/examples/field.cpp | 2 +- src/Core/iostream_debug_helpers.cpp | 3 +- .../CheckConstraintsBlockOutputStream.cpp | 2 +- .../CheckSortedBlockInputStream.cpp | 2 +- src/DataTypes/DataTypeAggregateFunction.cpp | 5 +- .../DataTypeCustomSimpleAggregateFunction.cpp | 6 +- .../getDictionaryConfigurationFromAST.cpp | 2 + src/Functions/FunctionBinaryArithmetic.h | 1 - src/Functions/FunctionsLogical.cpp | 2 +- src/Functions/GatherUtils/Algorithms.h | 2 +- src/Functions/abs.cpp | 2 +- src/Functions/intExp10.cpp | 2 +- src/Functions/intExp2.cpp | 2 +- src/Functions/sleep.h | 2 +- src/Functions/transform.cpp | 2 +- src/Interpreters/AggregateDescription.cpp | 3 +- src/Interpreters/CatBoostModel.cpp | 3 +- src/Interpreters/Context.cpp | 1 + src/Interpreters/FillingRow.cpp | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 1 + src/Interpreters/JIT/CompileDAG.cpp | 2 +- src/Interpreters/WindowDescription.cpp | 2 +- src/Parsers/ASTDictionary.cpp | 2 + src/Parsers/ASTFunction.cpp | 2 + src/Parsers/ASTLiteral.cpp | 3 +- src/Parsers/ASTLiteral.h | 2 +- src/Parsers/ASTSetQuery.cpp | 2 + src/Parsers/ASTSetQuery.h | 1 - src/Parsers/ASTSettingsProfileElement.cpp | 2 +- src/Parsers/ASTWindowDefinition.cpp | 1 - src/Parsers/ParserCreateQuotaQuery.cpp | 1 + src/Parsers/makeASTForLogicalFunction.cpp | 1 + .../Algorithms/CollapsingSortedAlgorithm.cpp | 3 +- .../Algorithms/SummingSortedAlgorithm.cpp | 2 +- src/Storages/MergeTree/KeyCondition.cpp | 1 + src/Storages/MergeTree/MergeTreePartition.cpp | 2 + src/Storages/StorageBuffer.cpp | 2 +- src/TableFunctions/TableFunctionNumbers.cpp | 1 + utils/db-generator/query_db_generator.cpp | 2 + 68 files changed, 892 insertions(+), 753 deletions(-) create mode 100644 src/Common/FieldVisitorConvertToNumber.h create mode 100644 src/Common/FieldVisitorDump.cpp create mode 100644 src/Common/FieldVisitorDump.h create mode 100644 src/Common/FieldVisitorHash.cpp create mode 100644 src/Common/FieldVisitorHash.h create mode 100644 src/Common/FieldVisitorSum.cpp create mode 100644 src/Common/FieldVisitorSum.h create mode 100644 src/Common/FieldVisitorToString.cpp create mode 100644 src/Common/FieldVisitorToString.h create mode 100644 src/Common/FieldVisitorWriteBinary.cpp create mode 100644 src/Common/FieldVisitorWriteBinary.h delete mode 100644 src/Common/FieldVisitors.cpp diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 958075541c8..316f869fc79 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h index b804e4465ac..861ef9cd292 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.cpp b/src/AggregateFunctions/AggregateFunctionHistogram.cpp index 4ff9b935f06..84650298ee6 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.cpp +++ b/src/AggregateFunctions/AggregateFunctionHistogram.cpp @@ -2,8 +2,7 @@ #include #include #include - -#include +#include namespace DB diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index 4e10316899a..12986598cb1 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include "AggregateFunctionFactory.h" diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index ec2f24d12cb..55d6544a4c1 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp index ba26247ce31..c3b80cae080 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.cpp +++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp b/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp index 21da94af4ae..8d1111519e9 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp b/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp index 280fbff4e7f..e417517ef6d 100644 --- a/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/AggregateFunctions/QuantilesCommon.h b/src/AggregateFunctions/QuantilesCommon.h index ab21a4c2efd..8a1645c3781 100644 --- a/src/AggregateFunctions/QuantilesCommon.h +++ b/src/AggregateFunctions/QuantilesCommon.h @@ -2,7 +2,7 @@ #include -#include +#include #include diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index fcaa5454db0..58e9bb05c1b 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h new file mode 100644 index 00000000000..0f099c6215d --- /dev/null +++ b/src/Common/FieldVisitorConvertToNumber.h @@ -0,0 +1,129 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_CONVERT_TYPE; + extern const int NOT_IMPLEMENTED; +} + + +/** Converts numeric value of any type to specified type. */ +template +class FieldVisitorConvertToNumber : public StaticVisitor +{ +public: + T operator() (const Null &) const + { + throw Exception("Cannot convert NULL to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const String &) const + { + throw Exception("Cannot convert String to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const Array &) const + { + throw Exception("Cannot convert Array to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const Tuple &) const + { + throw Exception("Cannot convert Tuple to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const Map &) const + { + throw Exception("Cannot convert Map to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const UInt64 & x) const { return T(x); } + T operator() (const Int64 & x) const { return T(x); } + T operator() (const Int128 & x) const { return T(x); } + T operator() (const UUID & x) const { return T(x.toUnderType()); } + + T operator() (const Float64 & x) const + { + if constexpr (!std::is_floating_point_v) + { + if (!isFinite(x)) + { + /// When converting to bool it's ok (non-zero converts to true, NaN including). + if (std::is_same_v) + return true; + + /// Conversion of infinite values to integer is undefined. + throw Exception("Cannot convert infinite value to integer type", ErrorCodes::CANNOT_CONVERT_TYPE); + } + else if (x > std::numeric_limits::max() || x < std::numeric_limits::lowest()) + { + throw Exception("Cannot convert out of range floating point value to integer type", ErrorCodes::CANNOT_CONVERT_TYPE); + } + } + + if constexpr (std::is_same_v) + { + return Int256(x); + } + else + { + return T(x); + } + } + + T operator() (const UInt128 &) const + { + throw Exception("Cannot convert UInt128 to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + template + T operator() (const DecimalField & x) const + { + if constexpr (std::is_floating_point_v) + return x.getValue(). template convertTo() / x.getScaleMultiplier(). template convertTo(); + else if constexpr (std::is_same_v) + { + if constexpr (sizeof(U) < 16) + { + return UInt128(0, (x.getValue() / x.getScaleMultiplier()).value); + } + else if constexpr (sizeof(U) == 16) + { + auto tmp = (x.getValue() / x.getScaleMultiplier()).value; + return UInt128(tmp >> 64, UInt64(tmp)); + } + else + throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); + } + else + return (x.getValue() / x.getScaleMultiplier()). template convertTo(); + } + + T operator() (const AggregateFunctionStateData &) const + { + throw Exception("Cannot convert AggregateFunctionStateData to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + template > > + T operator() (const U & x) const + { + if constexpr (IsDecimalNumber) + return static_cast(static_cast(x)); + else if constexpr (std::is_same_v) + throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); + else + return static_cast(x); + } +}; + +} + diff --git a/src/Common/FieldVisitorDump.cpp b/src/Common/FieldVisitorDump.cpp new file mode 100644 index 00000000000..e6726a4502e --- /dev/null +++ b/src/Common/FieldVisitorDump.cpp @@ -0,0 +1,109 @@ +#include + +#include +#include + + +namespace DB +{ + +template +static inline String formatQuotedWithPrefix(T x, const char * prefix) +{ + WriteBufferFromOwnString wb; + writeCString(prefix, wb); + writeQuoted(x, wb); + return wb.str(); +} + +template +static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) +{ + writeChar('\'', buf); + writeText(x.getValue(), x.getScale(), buf); + writeChar('\'', buf); +} + +String FieldVisitorDump::operator() (const Null &) const { return "NULL"; } +String FieldVisitorDump::operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); } +String FieldVisitorDump::operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); } +String FieldVisitorDump::operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); } +String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal32_"); } +String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); } +String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); } +String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); } +String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(x, "UInt128_"); } +String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); } +String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); } +String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); } +String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); } + + +String FieldVisitorDump::operator() (const String & x) const +{ + WriteBufferFromOwnString wb; + writeQuoted(x, wb); + return wb.str(); +} + +String FieldVisitorDump::operator() (const Array & x) const +{ + WriteBufferFromOwnString wb; + + wb << "Array_["; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ']'; + + return wb.str(); +} + +String FieldVisitorDump::operator() (const Tuple & x) const +{ + WriteBufferFromOwnString wb; + + wb << "Tuple_("; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + +String FieldVisitorDump::operator() (const Map & x) const +{ + WriteBufferFromOwnString wb; + + wb << "Map_("; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + +String FieldVisitorDump::operator() (const AggregateFunctionStateData & x) const +{ + WriteBufferFromOwnString wb; + wb << "AggregateFunctionState_("; + writeQuoted(x.name, wb); + wb << ", "; + writeQuoted(x.data, wb); + wb << ')'; + return wb.str(); +} + +} + diff --git a/src/Common/FieldVisitorDump.h b/src/Common/FieldVisitorDump.h new file mode 100644 index 00000000000..22e34d66ff7 --- /dev/null +++ b/src/Common/FieldVisitorDump.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace DB +{ + +/** Print readable and unique text dump of field type and value. */ +class FieldVisitorDump : public StaticVisitor +{ +public: + String operator() (const Null & x) const; + String operator() (const UInt64 & x) const; + String operator() (const UInt128 & x) const; + String operator() (const UInt256 & x) const; + String operator() (const Int64 & x) const; + String operator() (const Int128 & x) const; + String operator() (const Int256 & x) const; + String operator() (const UUID & x) const; + String operator() (const Float64 & x) const; + String operator() (const String & x) const; + String operator() (const Array & x) const; + String operator() (const Tuple & x) const; + String operator() (const Map & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const AggregateFunctionStateData & x) const; +}; + +} + diff --git a/src/Common/FieldVisitorHash.cpp b/src/Common/FieldVisitorHash.cpp new file mode 100644 index 00000000000..80d5f2daf65 --- /dev/null +++ b/src/Common/FieldVisitorHash.cpp @@ -0,0 +1,149 @@ +#include + +#include + + +namespace DB +{ + +FieldVisitorHash::FieldVisitorHash(SipHash & hash_) : hash(hash_) {} + +void FieldVisitorHash::operator() (const Null &) const +{ + UInt8 type = Field::Types::Null; + hash.update(type); +} + +void FieldVisitorHash::operator() (const UInt64 & x) const +{ + UInt8 type = Field::Types::UInt64; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const UInt128 & x) const +{ + UInt8 type = Field::Types::UInt128; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const Int64 & x) const +{ + UInt8 type = Field::Types::Int64; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const Int128 & x) const +{ + UInt8 type = Field::Types::Int128; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const UUID & x) const +{ + UInt8 type = Field::Types::UUID; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const Float64 & x) const +{ + UInt8 type = Field::Types::Float64; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const String & x) const +{ + UInt8 type = Field::Types::String; + hash.update(type); + hash.update(x.size()); + hash.update(x.data(), x.size()); +} + +void FieldVisitorHash::operator() (const Tuple & x) const +{ + UInt8 type = Field::Types::Tuple; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); +} + +void FieldVisitorHash::operator() (const Map & x) const +{ + UInt8 type = Field::Types::Map; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); +} + +void FieldVisitorHash::operator() (const Array & x) const +{ + UInt8 type = Field::Types::Array; + hash.update(type); + hash.update(x.size()); + + for (const auto & elem : x) + applyVisitor(*this, elem); +} + +void FieldVisitorHash::operator() (const DecimalField & x) const +{ + UInt8 type = Field::Types::Decimal32; + hash.update(type); + hash.update(x.getValue().value); +} + +void FieldVisitorHash::operator() (const DecimalField & x) const +{ + UInt8 type = Field::Types::Decimal64; + hash.update(type); + hash.update(x.getValue().value); +} + +void FieldVisitorHash::operator() (const DecimalField & x) const +{ + UInt8 type = Field::Types::Decimal128; + hash.update(type); + hash.update(x.getValue().value); +} + +void FieldVisitorHash::operator() (const DecimalField & x) const +{ + UInt8 type = Field::Types::Decimal256; + hash.update(type); + hash.update(x.getValue().value); +} + +void FieldVisitorHash::operator() (const AggregateFunctionStateData & x) const +{ + UInt8 type = Field::Types::AggregateFunctionState; + hash.update(type); + hash.update(x.name.size()); + hash.update(x.name.data(), x.name.size()); + hash.update(x.data.size()); + hash.update(x.data.data(), x.data.size()); +} + +void FieldVisitorHash::operator() (const UInt256 & x) const +{ + UInt8 type = Field::Types::UInt256; + hash.update(type); + hash.update(x); +} + +void FieldVisitorHash::operator() (const Int256 & x) const +{ + UInt8 type = Field::Types::Int256; + hash.update(type); + hash.update(x); +} + +} diff --git a/src/Common/FieldVisitorHash.h b/src/Common/FieldVisitorHash.h new file mode 100644 index 00000000000..6c786fda4ad --- /dev/null +++ b/src/Common/FieldVisitorHash.h @@ -0,0 +1,38 @@ +#pragma once + +#include + +class SipHash; + +namespace DB +{ + +/** Updates SipHash by type and value of Field */ +class FieldVisitorHash : public StaticVisitor<> +{ +private: + SipHash & hash; +public: + FieldVisitorHash(SipHash & hash_); + + void operator() (const Null & x) const; + void operator() (const UInt64 & x) const; + void operator() (const UInt128 & x) const; + void operator() (const UInt256 & x) const; + void operator() (const Int64 & x) const; + void operator() (const Int128 & x) const; + void operator() (const Int256 & x) const; + void operator() (const UUID & x) const; + void operator() (const Float64 & x) const; + void operator() (const String & x) const; + void operator() (const Array & x) const; + void operator() (const Tuple & x) const; + void operator() (const Map & x) const; + void operator() (const DecimalField & x) const; + void operator() (const DecimalField & x) const; + void operator() (const DecimalField & x) const; + void operator() (const DecimalField & x) const; + void operator() (const AggregateFunctionStateData & x) const; +}; + +} diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp new file mode 100644 index 00000000000..0064830c08a --- /dev/null +++ b/src/Common/FieldVisitorSum.cpp @@ -0,0 +1,37 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +FieldVisitorSum::FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} + +// We can add all ints as unsigned regardless of their actual signedness. +bool FieldVisitorSum::operator() (Int64 & x) const { return this->operator()(reinterpret_cast(x)); } +bool FieldVisitorSum::operator() (UInt64 & x) const +{ + x += rhs.reinterpret(); + return x != 0; +} + +bool FieldVisitorSum::operator() (Float64 & x) const { x += get(rhs); return x != 0; } + +bool FieldVisitorSum::operator() (Null &) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (String &) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (UUID &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); } + +bool FieldVisitorSum::operator() (AggregateFunctionStateData &) const +{ + throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); +} + +} + diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h new file mode 100644 index 00000000000..e208933043b --- /dev/null +++ b/src/Common/FieldVisitorSum.h @@ -0,0 +1,47 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Implements `+=` operation. + * Returns false if the result is zero. + */ +class FieldVisitorSum : public StaticVisitor +{ +private: + const Field & rhs; +public: + explicit FieldVisitorSum(const Field & rhs_); + + // We can add all ints as unsigned regardless of their actual signedness. + bool operator() (Int64 & x) const; + bool operator() (UInt64 & x) const; + bool operator() (Float64 & x) const; + bool operator() (Null &) const; + bool operator() (String &) const; + bool operator() (Array &) const; + bool operator() (Tuple &) const; + bool operator() (Map &) const; + bool operator() (UUID &) const; + bool operator() (AggregateFunctionStateData &) const; + + template + bool operator() (DecimalField & x) const + { + x += get>(rhs); + return x.getValue() != T(0); + } + + template > > + bool operator() (T & x) const + { + x += rhs.reinterpret(); + return x != T(0); + } +}; + +} + diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp new file mode 100644 index 00000000000..45bc54f2c2a --- /dev/null +++ b/src/Common/FieldVisitorToString.cpp @@ -0,0 +1,130 @@ +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER; +} + + +template +static inline String formatQuoted(T x) +{ + WriteBufferFromOwnString wb; + writeQuoted(x, wb); + return wb.str(); +} + +template +static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) +{ + writeChar('\'', buf); + writeText(x.getValue(), x.getScale(), buf); + writeChar('\'', buf); +} + +/** In contrast to writeFloatText (and writeQuoted), + * even if number looks like integer after formatting, prints decimal point nevertheless (for example, Float64(1) is printed as 1.). + * - because resulting text must be able to be parsed back as Float64 by query parser (otherwise it will be parsed as integer). + * + * Trailing zeros after decimal point are omitted. + * + * NOTE: Roundtrip may lead to loss of precision. + */ +static String formatFloat(const Float64 x) +{ + DoubleConverter::BufferType buffer; + double_conversion::StringBuilder builder{buffer, sizeof(buffer)}; + + const auto result = DoubleConverter::instance().ToShortest(x, &builder); + + if (!result) + throw Exception("Cannot print float or double number", ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER); + + return { buffer, buffer + builder.position() }; +} + + +String FieldVisitorToString::operator() (const Null &) const { return "NULL"; } +String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const Float64 & x) const { return formatFloat(x); } +String FieldVisitorToString::operator() (const String & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const Int128 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); } +String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); } + +String FieldVisitorToString::operator() (const Array & x) const +{ + WriteBufferFromOwnString wb; + + wb << '['; + for (Array::const_iterator it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb.write(", ", 2); + wb << applyVisitor(*this, *it); + } + wb << ']'; + + return wb.str(); +} + +String FieldVisitorToString::operator() (const Tuple & x) const +{ + WriteBufferFromOwnString wb; + + // For single-element tuples we must use the explicit tuple() function, + // or they will be parsed back as plain literals. + if (x.size() > 1) + { + wb << '('; + } + else + { + wb << "tuple("; + } + + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + +String FieldVisitorToString::operator() (const Map & x) const +{ + WriteBufferFromOwnString wb; + + wb << '('; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + +} + diff --git a/src/Common/FieldVisitorToString.h b/src/Common/FieldVisitorToString.h new file mode 100644 index 00000000000..39709f1c272 --- /dev/null +++ b/src/Common/FieldVisitorToString.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace DB +{ + +/** Prints Field as literal in SQL query */ +class FieldVisitorToString : public StaticVisitor +{ +public: + String operator() (const Null & x) const; + String operator() (const UInt64 & x) const; + String operator() (const UInt128 & x) const; + String operator() (const UInt256 & x) const; + String operator() (const Int64 & x) const; + String operator() (const Int128 & x) const; + String operator() (const Int256 & x) const; + String operator() (const UUID & x) const; + String operator() (const Float64 & x) const; + String operator() (const String & x) const; + String operator() (const Array & x) const; + String operator() (const Tuple & x) const; + String operator() (const Map & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const DecimalField & x) const; + String operator() (const AggregateFunctionStateData & x) const; +}; + +} + diff --git a/src/Common/FieldVisitorWriteBinary.cpp b/src/Common/FieldVisitorWriteBinary.cpp new file mode 100644 index 00000000000..8e991ad13d3 --- /dev/null +++ b/src/Common/FieldVisitorWriteBinary.cpp @@ -0,0 +1,70 @@ +#include + +#include + + +namespace DB +{ + +void FieldVisitorWriteBinary::operator() (const Null &, WriteBuffer &) const { } +void FieldVisitorWriteBinary::operator() (const UInt64 & x, WriteBuffer & buf) const { writeVarUInt(x, buf); } +void FieldVisitorWriteBinary::operator() (const Int64 & x, WriteBuffer & buf) const { writeVarInt(x, buf); } +void FieldVisitorWriteBinary::operator() (const Float64 & x, WriteBuffer & buf) const { writeFloatBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const String & x, WriteBuffer & buf) const { writeStringBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const UInt128 & x, WriteBuffer & buf) const { writeBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) const { writeVarInt(x, buf); } +void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { writeBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { writeBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { writeBinary(x, buf); } +void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); } +void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); } +void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); } +void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); } +void FieldVisitorWriteBinary::operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const +{ + writeStringBinary(x.name, buf); + writeStringBinary(x.data, buf); +} + +void FieldVisitorWriteBinary::operator() (const Array & x, WriteBuffer & buf) const +{ + const size_t size = x.size(); + writeBinary(size, buf); + + for (size_t i = 0; i < size; ++i) + { + const UInt8 type = x[i].getType(); + writeBinary(type, buf); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, x[i]); + } +} + +void FieldVisitorWriteBinary::operator() (const Tuple & x, WriteBuffer & buf) const +{ + const size_t size = x.size(); + writeBinary(size, buf); + + for (size_t i = 0; i < size; ++i) + { + const UInt8 type = x[i].getType(); + writeBinary(type, buf); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, x[i]); + } +} + + +void FieldVisitorWriteBinary::operator() (const Map & x, WriteBuffer & buf) const +{ + const size_t size = x.size(); + writeBinary(size, buf); + + for (size_t i = 0; i < size; ++i) + { + const UInt8 type = x[i].getType(); + writeBinary(type, buf); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, x[i]); + } +} + +} + diff --git a/src/Common/FieldVisitorWriteBinary.h b/src/Common/FieldVisitorWriteBinary.h new file mode 100644 index 00000000000..ae864ca74f3 --- /dev/null +++ b/src/Common/FieldVisitorWriteBinary.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace DB +{ + +class FieldVisitorWriteBinary +{ +public: + void operator() (const Null & x, WriteBuffer & buf) const; + void operator() (const UInt64 & x, WriteBuffer & buf) const; + void operator() (const UInt128 & x, WriteBuffer & buf) const; + void operator() (const UInt256 & x, WriteBuffer & buf) const; + void operator() (const Int64 & x, WriteBuffer & buf) const; + void operator() (const Int128 & x, WriteBuffer & buf) const; + void operator() (const Int256 & x, WriteBuffer & buf) const; + void operator() (const UUID & x, WriteBuffer & buf) const; + void operator() (const Float64 & x, WriteBuffer & buf) const; + void operator() (const String & x, WriteBuffer & buf) const; + void operator() (const Array & x, WriteBuffer & buf) const; + void operator() (const Tuple & x, WriteBuffer & buf) const; + void operator() (const Map & x, WriteBuffer & buf) const; + void operator() (const DecimalField & x, WriteBuffer & buf) const; + void operator() (const DecimalField & x, WriteBuffer & buf) const; + void operator() (const DecimalField & x, WriteBuffer & buf) const; + void operator() (const DecimalField & x, WriteBuffer & buf) const; + void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const; +}; + +} + diff --git a/src/Common/FieldVisitors.cpp b/src/Common/FieldVisitors.cpp deleted file mode 100644 index b87152da7e7..00000000000 --- a/src/Common/FieldVisitors.cpp +++ /dev/null @@ -1,421 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER; -} - -template -static inline String formatQuoted(T x) -{ - WriteBufferFromOwnString wb; - writeQuoted(x, wb); - return wb.str(); -} - -template -static inline String formatQuotedWithPrefix(T x, const char * prefix) -{ - WriteBufferFromOwnString wb; - writeCString(prefix, wb); - writeQuoted(x, wb); - return wb.str(); -} - -template -static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) -{ - writeChar('\'', buf); - writeText(x.getValue(), x.getScale(), buf); - writeChar('\'', buf); -} - -String FieldVisitorDump::operator() (const Null &) const { return "NULL"; } -String FieldVisitorDump::operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); } -String FieldVisitorDump::operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); } -String FieldVisitorDump::operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); } -String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal32_"); } -String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); } -String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); } -String FieldVisitorDump::operator() (const DecimalField & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); } -String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(x, "UInt128_"); } -String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); } -String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); } -String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); } -String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); } - - -String FieldVisitorDump::operator() (const String & x) const -{ - WriteBufferFromOwnString wb; - writeQuoted(x, wb); - return wb.str(); -} - -String FieldVisitorDump::operator() (const Array & x) const -{ - WriteBufferFromOwnString wb; - - wb << "Array_["; - for (auto it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb << ", "; - wb << applyVisitor(*this, *it); - } - wb << ']'; - - return wb.str(); -} - -String FieldVisitorDump::operator() (const Tuple & x) const -{ - WriteBufferFromOwnString wb; - - wb << "Tuple_("; - for (auto it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb << ", "; - wb << applyVisitor(*this, *it); - } - wb << ')'; - - return wb.str(); -} - -String FieldVisitorDump::operator() (const Map & x) const -{ - WriteBufferFromOwnString wb; - - wb << "Map_("; - for (auto it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb << ", "; - wb << applyVisitor(*this, *it); - } - wb << ')'; - - return wb.str(); -} - -String FieldVisitorDump::operator() (const AggregateFunctionStateData & x) const -{ - WriteBufferFromOwnString wb; - wb << "AggregateFunctionState_("; - writeQuoted(x.name, wb); - wb << ", "; - writeQuoted(x.data, wb); - wb << ')'; - return wb.str(); -} - -/** In contrast to writeFloatText (and writeQuoted), - * even if number looks like integer after formatting, prints decimal point nevertheless (for example, Float64(1) is printed as 1.). - * - because resulting text must be able to be parsed back as Float64 by query parser (otherwise it will be parsed as integer). - * - * Trailing zeros after decimal point are omitted. - * - * NOTE: Roundtrip may lead to loss of precision. - */ -static String formatFloat(const Float64 x) -{ - DoubleConverter::BufferType buffer; - double_conversion::StringBuilder builder{buffer, sizeof(buffer)}; - - const auto result = DoubleConverter::instance().ToShortest(x, &builder); - - if (!result) - throw Exception("Cannot print float or double number", ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER); - - return { buffer, buffer + builder.position() }; -} - - -String FieldVisitorToString::operator() (const Null &) const { return "NULL"; } -String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const Float64 & x) const { return formatFloat(x); } -String FieldVisitorToString::operator() (const String & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const DecimalField & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const Int128 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); } -String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); } - -String FieldVisitorToString::operator() (const Array & x) const -{ - WriteBufferFromOwnString wb; - - wb << '['; - for (Array::const_iterator it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb.write(", ", 2); - wb << applyVisitor(*this, *it); - } - wb << ']'; - - return wb.str(); -} - -String FieldVisitorToString::operator() (const Tuple & x) const -{ - WriteBufferFromOwnString wb; - - // For single-element tuples we must use the explicit tuple() function, - // or they will be parsed back as plain literals. - if (x.size() > 1) - { - wb << '('; - } - else - { - wb << "tuple("; - } - - for (auto it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb << ", "; - wb << applyVisitor(*this, *it); - } - wb << ')'; - - return wb.str(); -} - -String FieldVisitorToString::operator() (const Map & x) const -{ - WriteBufferFromOwnString wb; - - wb << '('; - for (auto it = x.begin(); it != x.end(); ++it) - { - if (it != x.begin()) - wb << ", "; - wb << applyVisitor(*this, *it); - } - wb << ')'; - - return wb.str(); -} - - -void FieldVisitorWriteBinary::operator() (const Null &, WriteBuffer &) const { } -void FieldVisitorWriteBinary::operator() (const UInt64 & x, WriteBuffer & buf) const { DB::writeVarUInt(x, buf); } -void FieldVisitorWriteBinary::operator() (const Int64 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); } -void FieldVisitorWriteBinary::operator() (const Float64 & x, WriteBuffer & buf) const { DB::writeFloatBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const String & x, WriteBuffer & buf) const { DB::writeStringBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const UInt128 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); } -void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); } -void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); } -void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); } -void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); } -void FieldVisitorWriteBinary::operator() (const DecimalField & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); } -void FieldVisitorWriteBinary::operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const -{ - DB::writeStringBinary(x.name, buf); - DB::writeStringBinary(x.data, buf); -} - -void FieldVisitorWriteBinary::operator() (const Array & x, WriteBuffer & buf) const -{ - const size_t size = x.size(); - DB::writeBinary(size, buf); - - for (size_t i = 0; i < size; ++i) - { - const UInt8 type = x[i].getType(); - DB::writeBinary(type, buf); - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]); - } -} - -void FieldVisitorWriteBinary::operator() (const Tuple & x, WriteBuffer & buf) const -{ - const size_t size = x.size(); - DB::writeBinary(size, buf); - - for (size_t i = 0; i < size; ++i) - { - const UInt8 type = x[i].getType(); - DB::writeBinary(type, buf); - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]); - } -} - - -void FieldVisitorWriteBinary::operator() (const Map & x, WriteBuffer & buf) const -{ - const size_t size = x.size(); - DB::writeBinary(size, buf); - - for (size_t i = 0; i < size; ++i) - { - const UInt8 type = x[i].getType(); - writeBinary(type, buf); - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, x[i]); - } -} - - -FieldVisitorHash::FieldVisitorHash(SipHash & hash_) : hash(hash_) {} - -void FieldVisitorHash::operator() (const Null &) const -{ - UInt8 type = Field::Types::Null; - hash.update(type); -} - -void FieldVisitorHash::operator() (const UInt64 & x) const -{ - UInt8 type = Field::Types::UInt64; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const UInt128 & x) const -{ - UInt8 type = Field::Types::UInt128; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const Int64 & x) const -{ - UInt8 type = Field::Types::Int64; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const Int128 & x) const -{ - UInt8 type = Field::Types::Int128; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const UUID & x) const -{ - UInt8 type = Field::Types::UUID; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const Float64 & x) const -{ - UInt8 type = Field::Types::Float64; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const String & x) const -{ - UInt8 type = Field::Types::String; - hash.update(type); - hash.update(x.size()); - hash.update(x.data(), x.size()); -} - -void FieldVisitorHash::operator() (const Tuple & x) const -{ - UInt8 type = Field::Types::Tuple; - hash.update(type); - hash.update(x.size()); - - for (const auto & elem : x) - applyVisitor(*this, elem); -} - -void FieldVisitorHash::operator() (const Map & x) const -{ - UInt8 type = Field::Types::Map; - hash.update(type); - hash.update(x.size()); - - for (const auto & elem : x) - applyVisitor(*this, elem); -} - -void FieldVisitorHash::operator() (const Array & x) const -{ - UInt8 type = Field::Types::Array; - hash.update(type); - hash.update(x.size()); - - for (const auto & elem : x) - applyVisitor(*this, elem); -} - -void FieldVisitorHash::operator() (const DecimalField & x) const -{ - UInt8 type = Field::Types::Decimal32; - hash.update(type); - hash.update(x.getValue().value); -} - -void FieldVisitorHash::operator() (const DecimalField & x) const -{ - UInt8 type = Field::Types::Decimal64; - hash.update(type); - hash.update(x.getValue().value); -} - -void FieldVisitorHash::operator() (const DecimalField & x) const -{ - UInt8 type = Field::Types::Decimal128; - hash.update(type); - hash.update(x.getValue().value); -} - -void FieldVisitorHash::operator() (const DecimalField & x) const -{ - UInt8 type = Field::Types::Decimal256; - hash.update(type); - hash.update(x.getValue().value); -} - -void FieldVisitorHash::operator() (const AggregateFunctionStateData & x) const -{ - UInt8 type = Field::Types::AggregateFunctionState; - hash.update(type); - hash.update(x.name.size()); - hash.update(x.name.data(), x.name.size()); - hash.update(x.data.size()); - hash.update(x.data.data(), x.data.size()); -} - -void FieldVisitorHash::operator() (const UInt256 & x) const -{ - UInt8 type = Field::Types::UInt256; - hash.update(type); - hash.update(x); -} - -void FieldVisitorHash::operator() (const Int256 & x) const -{ - UInt8 type = Field::Types::Int256; - hash.update(type); - hash.update(x); -} - -} diff --git a/src/Common/FieldVisitors.h b/src/Common/FieldVisitors.h index 2f7e24e4fad..99c4c42360b 100644 --- a/src/Common/FieldVisitors.h +++ b/src/Common/FieldVisitors.h @@ -1,25 +1,11 @@ #pragma once -#include #include -#include -#include - - -class SipHash; namespace DB { -namespace ErrorCodes -{ - extern const int CANNOT_CONVERT_TYPE; - extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; -} - - /** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields. * You could call visitor for field using function 'applyVisitor'. * Also "binary visitor" is supported - its operator() takes two arguments. @@ -55,268 +41,4 @@ auto applyVisitor(Visitor && visitor, F1 && field1, F2 && field2) std::forward(field1)); } - -/** Prints Field as literal in SQL query */ -class FieldVisitorToString : public StaticVisitor -{ -public: - String operator() (const Null & x) const; - String operator() (const UInt64 & x) const; - String operator() (const UInt128 & x) const; - String operator() (const UInt256 & x) const; - String operator() (const Int64 & x) const; - String operator() (const Int128 & x) const; - String operator() (const Int256 & x) const; - String operator() (const UUID & x) const; - String operator() (const Float64 & x) const; - String operator() (const String & x) const; - String operator() (const Array & x) const; - String operator() (const Tuple & x) const; - String operator() (const Map & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const AggregateFunctionStateData & x) const; -}; - - -class FieldVisitorWriteBinary -{ -public: - void operator() (const Null & x, WriteBuffer & buf) const; - void operator() (const UInt64 & x, WriteBuffer & buf) const; - void operator() (const UInt128 & x, WriteBuffer & buf) const; - void operator() (const UInt256 & x, WriteBuffer & buf) const; - void operator() (const Int64 & x, WriteBuffer & buf) const; - void operator() (const Int128 & x, WriteBuffer & buf) const; - void operator() (const Int256 & x, WriteBuffer & buf) const; - void operator() (const UUID & x, WriteBuffer & buf) const; - void operator() (const Float64 & x, WriteBuffer & buf) const; - void operator() (const String & x, WriteBuffer & buf) const; - void operator() (const Array & x, WriteBuffer & buf) const; - void operator() (const Tuple & x, WriteBuffer & buf) const; - void operator() (const Map & x, WriteBuffer & buf) const; - void operator() (const DecimalField & x, WriteBuffer & buf) const; - void operator() (const DecimalField & x, WriteBuffer & buf) const; - void operator() (const DecimalField & x, WriteBuffer & buf) const; - void operator() (const DecimalField & x, WriteBuffer & buf) const; - void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const; -}; - - -/** Print readable and unique text dump of field type and value. */ -class FieldVisitorDump : public StaticVisitor -{ -public: - String operator() (const Null & x) const; - String operator() (const UInt64 & x) const; - String operator() (const UInt128 & x) const; - String operator() (const UInt256 & x) const; - String operator() (const Int64 & x) const; - String operator() (const Int128 & x) const; - String operator() (const Int256 & x) const; - String operator() (const UUID & x) const; - String operator() (const Float64 & x) const; - String operator() (const String & x) const; - String operator() (const Array & x) const; - String operator() (const Tuple & x) const; - String operator() (const Map & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const DecimalField & x) const; - String operator() (const AggregateFunctionStateData & x) const; -}; - - -/** Converts numeric value of any type to specified type. */ -template -class FieldVisitorConvertToNumber : public StaticVisitor -{ -public: - T operator() (const Null &) const - { - throw Exception("Cannot convert NULL to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - T operator() (const String &) const - { - throw Exception("Cannot convert String to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - T operator() (const Array &) const - { - throw Exception("Cannot convert Array to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - T operator() (const Tuple &) const - { - throw Exception("Cannot convert Tuple to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - T operator() (const Map &) const - { - throw Exception("Cannot convert Map to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - T operator() (const UInt64 & x) const { return T(x); } - T operator() (const Int64 & x) const { return T(x); } - T operator() (const Int128 & x) const { return T(x); } - T operator() (const UUID & x) const { return T(x.toUnderType()); } - - T operator() (const Float64 & x) const - { - if constexpr (!std::is_floating_point_v) - { - if (!isFinite(x)) - { - /// When converting to bool it's ok (non-zero converts to true, NaN including). - if (std::is_same_v) - return true; - - /// Conversion of infinite values to integer is undefined. - throw Exception("Cannot convert infinite value to integer type", ErrorCodes::CANNOT_CONVERT_TYPE); - } - else if (x > std::numeric_limits::max() || x < std::numeric_limits::lowest()) - { - throw Exception("Cannot convert out of range floating point value to integer type", ErrorCodes::CANNOT_CONVERT_TYPE); - } - } - - if constexpr (std::is_same_v) - { - return Int256(x); - } - else - { - return T(x); - } - } - - T operator() (const UInt128 &) const - { - throw Exception("Cannot convert UInt128 to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - template - T operator() (const DecimalField & x) const - { - if constexpr (std::is_floating_point_v) - return x.getValue(). template convertTo() / x.getScaleMultiplier(). template convertTo(); - else if constexpr (std::is_same_v) - { - /// TODO: remove with old UInt128 type - if constexpr (sizeof(U) < 16) - { - return UInt128(0, (x.getValue() / x.getScaleMultiplier()).value); - } - else if constexpr (sizeof(U) == 16) - { - auto tmp = (x.getValue() / x.getScaleMultiplier()).value; - return UInt128(tmp >> 64, UInt64(tmp)); - } - else - throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); - } - else - return (x.getValue() / x.getScaleMultiplier()). template convertTo(); - } - - T operator() (const AggregateFunctionStateData &) const - { - throw Exception("Cannot convert AggregateFunctionStateData to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); - } - - template > > - T operator() (const U & x) const - { - if constexpr (IsDecimalNumber) - return static_cast(static_cast(x)); - else if constexpr (std::is_same_v) - throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED); - else - return static_cast(x); - } -}; - - -/** Updates SipHash by type and value of Field */ -class FieldVisitorHash : public StaticVisitor<> -{ -private: - SipHash & hash; -public: - FieldVisitorHash(SipHash & hash_); - - void operator() (const Null & x) const; - void operator() (const UInt64 & x) const; - void operator() (const UInt128 & x) const; - void operator() (const UInt256 & x) const; - void operator() (const Int64 & x) const; - void operator() (const Int128 & x) const; - void operator() (const Int256 & x) const; - void operator() (const UUID & x) const; - void operator() (const Float64 & x) const; - void operator() (const String & x) const; - void operator() (const Array & x) const; - void operator() (const Tuple & x) const; - void operator() (const Map & x) const; - void operator() (const DecimalField & x) const; - void operator() (const DecimalField & x) const; - void operator() (const DecimalField & x) const; - void operator() (const DecimalField & x) const; - void operator() (const AggregateFunctionStateData & x) const; -}; - - -/** Implements `+=` operation. - * Returns false if the result is zero. - */ -class FieldVisitorSum : public StaticVisitor -{ -private: - const Field & rhs; -public: - explicit FieldVisitorSum(const Field & rhs_) : rhs(rhs_) {} - - // We can add all ints as unsigned regardless of their actual signedness. - bool operator() (Int64 & x) const { return this->operator()(reinterpret_cast(x)); } - bool operator() (UInt64 & x) const - { - x += rhs.reinterpret(); - return x != 0; - } - - bool operator() (Float64 & x) const { x += get(rhs); return x != 0; } - - bool operator() (Null &) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); } - bool operator() (String &) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); } - bool operator() (UUID &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); } - bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } - - bool operator() (Int128 & x) const - { - x += get(rhs); - return x != Int128(0); - } - - template - bool operator() (DecimalField & x) const - { - x += get>(rhs); - return x.getValue() != T(0); - } - - template > > - bool operator() (T & x) const - { - x += rhs.reinterpret(); - return x != T(0); - } -}; - } diff --git a/src/Common/ya.make b/src/Common/ya.make index 4f47eddd4f3..752d00e6cb5 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -44,7 +44,11 @@ SRCS( ErrorCodes.cpp Exception.cpp ExternalLoaderStatus.cpp - FieldVisitors.cpp + FieldVisitorDump.cpp + FieldVisitorHash.cpp + FieldVisitorSum.cpp + FieldVisitorToString.cpp + FieldVisitorWriteBinary.cpp FileChecker.cpp IO.cpp IPv6ToBinary.cpp diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 2d57f49000d..fa78f052f37 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 9dec8563d36..e625c92f826 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -6,7 +6,9 @@ #include #include #include -#include +#include +#include +#include namespace DB @@ -95,12 +97,12 @@ void writeBinary(const Array & x, WriteBuffer & buf) DB::writeBinary(size, buf); for (const auto & elem : x) - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); } void writeText(const Array & x, WriteBuffer & buf) { - DB::String res = applyVisitor(DB::FieldVisitorToString(), DB::Field(x)); + DB::String res = applyVisitor(FieldVisitorToString(), DB::Field(x)); buf.write(res.data(), res.size()); } @@ -126,7 +128,7 @@ void writeBinary(const Tuple & x, WriteBuffer & buf) { const UInt8 type = elem.getType(); DB::writeBinary(type, buf); - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); } } @@ -157,7 +159,7 @@ void writeBinary(const Map & x, WriteBuffer & buf) { const UInt8 type = elem.getType(); DB::writeBinary(type, buf); - Field::dispatch([&buf] (const auto & value) { DB::FieldVisitorWriteBinary()(value, buf); }, elem); + Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); } } @@ -194,14 +196,14 @@ template void readQuoted(DecimalField & x, ReadBuffer & void writeFieldText(const Field & x, WriteBuffer & buf) { - DB::String res = Field::dispatch(DB::FieldVisitorToString(), x); + String res = Field::dispatch(FieldVisitorToString(), x); buf.write(res.data(), res.size()); } String Field::dump() const { - return applyVisitor(DB::FieldVisitorDump(), *this); + return applyVisitor(FieldVisitorDump(), *this); } Field Field::restoreFromDump(const std::string_view & dump_) diff --git a/src/Core/Field.h b/src/Core/Field.h index 2242e8fddae..23569f5f9f1 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -163,10 +163,10 @@ private: #endif template constexpr bool is_decimal_field = false; -template <> constexpr bool is_decimal_field> = true; -template <> constexpr bool is_decimal_field> = true; -template <> constexpr bool is_decimal_field> = true; -template <> constexpr bool is_decimal_field> = true; +template <> constexpr inline bool is_decimal_field> = true; +template <> constexpr inline bool is_decimal_field> = true; +template <> constexpr inline bool is_decimal_field> = true; +template <> constexpr inline bool is_decimal_field> = true; /// char may be signed or unsigned, and behave identically to signed char or unsigned char, /// but they are always three different types. diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index e326d5e5b32..cb8cdf05c68 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -6,10 +6,11 @@ #include #include #include -#include +#include #include #include + namespace DB { namespace ErrorCodes diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index a9963ec5748..379c65a913f 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Core/examples/field.cpp b/src/Core/examples/field.cpp index b0a1c1151a6..3190a7fcb7d 100644 --- a/src/Core/examples/field.cpp +++ b/src/Core/examples/field.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/src/Core/iostream_debug_helpers.cpp b/src/Core/iostream_debug_helpers.cpp index 8dc8a4244ac..8ec06af049e 100644 --- a/src/Core/iostream_debug_helpers.cpp +++ b/src/Core/iostream_debug_helpers.cpp @@ -15,7 +15,8 @@ #include #include #include -#include +#include + namespace DB { diff --git a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp index c4556162323..fbf4a777032 100644 --- a/src/DataStreams/CheckConstraintsBlockOutputStream.cpp +++ b/src/DataStreams/CheckConstraintsBlockOutputStream.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/src/DataStreams/CheckSortedBlockInputStream.cpp b/src/DataStreams/CheckSortedBlockInputStream.cpp index 99026e72540..064c1b690b8 100644 --- a/src/DataStreams/CheckSortedBlockInputStream.cpp +++ b/src/DataStreams/CheckSortedBlockInputStream.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 11fdf4e6894..904d4d2745e 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -47,7 +48,7 @@ std::string DataTypeAggregateFunction::doGetName() const { if (i) stream << ", "; - stream << applyVisitor(DB::FieldVisitorToString(), parameters[i]); + stream << applyVisitor(FieldVisitorToString(), parameters[i]); } stream << ')'; } diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index 6b2f94aa7b5..023629fc699 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -13,6 +13,7 @@ #include + namespace DB { @@ -27,6 +28,7 @@ namespace ErrorCodes void DataTypeCustomSimpleAggregateFunction::checkSupportedFunctions(const AggregateFunctionPtr & function) { + /// TODO Make it sane. static const std::vector supported_functions{"any", "anyLast", "min", "max", "sum", "sumWithOverflow", "groupBitAnd", "groupBitOr", "groupBitXor", "sumMap", "minMap", "maxMap", "groupArrayArray", "groupUniqArrayArray"}; @@ -51,7 +53,7 @@ String DataTypeCustomSimpleAggregateFunction::getName() const { if (i) stream << ", "; - stream << applyVisitor(DB::FieldVisitorToString(), parameters[i]); + stream << applyVisitor(FieldVisitorToString(), parameters[i]); } stream << ")"; } diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index 2063ebcbf79..ba81b1f1364 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include #include + namespace DB { diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index c8cd8536f3a..06d945eb3ce 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -32,7 +32,6 @@ #include #include #include -#include #include #include diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 68eed88e59c..3806ee7511c 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index d17ab082004..2df2e988ec5 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include "Sources.h" #include "Sinks.h" #include diff --git a/src/Functions/abs.cpp b/src/Functions/abs.cpp index f0c530e0e8f..10664f7b421 100644 --- a/src/Functions/abs.cpp +++ b/src/Functions/abs.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/src/Functions/intExp10.cpp b/src/Functions/intExp10.cpp index b1964701ad7..daf87e717e9 100644 --- a/src/Functions/intExp10.cpp +++ b/src/Functions/intExp10.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include namespace DB diff --git a/src/Functions/intExp2.cpp b/src/Functions/intExp2.cpp index c87a6e31852..d09f51a1269 100644 --- a/src/Functions/intExp2.cpp +++ b/src/Functions/intExp2.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include namespace DB diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index c0aad0b3820..8f78fd19a1f 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 1debc2cb6a0..b886a3794f5 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index ca10878a4ae..1a0748b5f97 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -1,9 +1,10 @@ #include -#include +#include #include #include + namespace DB { diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp index e19258540b9..1b6e30a0959 100644 --- a/src/Interpreters/CatBoostModel.cpp +++ b/src/Interpreters/CatBoostModel.cpp @@ -1,6 +1,6 @@ #include "CatBoostModel.h" -#include +#include #include #include #include @@ -14,6 +14,7 @@ #include #include + namespace DB { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 12110b074eb..1fe1e32a2e1 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index 7e32d9514a6..4bbb8974fe9 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -1,4 +1,5 @@ #include +#include #include diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 85b9026c642..e4f91e62fb0 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -79,6 +79,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Interpreters/JIT/CompileDAG.cpp b/src/Interpreters/JIT/CompileDAG.cpp index 5fc88e7884c..2c5c7731150 100644 --- a/src/Interpreters/JIT/CompileDAG.cpp +++ b/src/Interpreters/JIT/CompileDAG.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Interpreters/WindowDescription.cpp b/src/Interpreters/WindowDescription.cpp index b9f8597706e..46e1eb12dc5 100644 --- a/src/Interpreters/WindowDescription.cpp +++ b/src/Interpreters/WindowDescription.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include diff --git a/src/Parsers/ASTDictionary.cpp b/src/Parsers/ASTDictionary.cpp index 3d6750f2336..66c1c3791b8 100644 --- a/src/Parsers/ASTDictionary.cpp +++ b/src/Parsers/ASTDictionary.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace DB { diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 8df3383f487..e8c3775d187 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include + namespace DB { diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp index ed6790499fb..f7947fe7a24 100644 --- a/src/Parsers/ASTLiteral.cpp +++ b/src/Parsers/ASTLiteral.cpp @@ -1,5 +1,6 @@ #include -#include +#include +#include #include #include #include diff --git a/src/Parsers/ASTLiteral.h b/src/Parsers/ASTLiteral.h index 7e472a16bdd..66d013d78a9 100644 --- a/src/Parsers/ASTLiteral.h +++ b/src/Parsers/ASTLiteral.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index c8a2b3b37e8..f6b3609b349 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include diff --git a/src/Parsers/ASTSetQuery.h b/src/Parsers/ASTSetQuery.h index a91584910bb..40a0b679650 100644 --- a/src/Parsers/ASTSetQuery.h +++ b/src/Parsers/ASTSetQuery.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include diff --git a/src/Parsers/ASTSettingsProfileElement.cpp b/src/Parsers/ASTSettingsProfileElement.cpp index 2422126219f..8f35c154a79 100644 --- a/src/Parsers/ASTSettingsProfileElement.cpp +++ b/src/Parsers/ASTSettingsProfileElement.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include diff --git a/src/Parsers/ASTWindowDefinition.cpp b/src/Parsers/ASTWindowDefinition.cpp index 7a91097ce99..613dbcc392b 100644 --- a/src/Parsers/ASTWindowDefinition.cpp +++ b/src/Parsers/ASTWindowDefinition.cpp @@ -1,7 +1,6 @@ #include #include -#include #include diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index a8779a68600..c2e1514c7f6 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/src/Parsers/makeASTForLogicalFunction.cpp b/src/Parsers/makeASTForLogicalFunction.cpp index 02c9da926c9..a1816dc9d17 100644 --- a/src/Parsers/makeASTForLogicalFunction.cpp +++ b/src/Parsers/makeASTForLogicalFunction.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include diff --git a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp index 0db99fc7b0e..2d873aa6bc6 100644 --- a/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/CollapsingSortedAlgorithm.cpp @@ -1,13 +1,14 @@ #include #include -#include +#include #include #include #include #include + /// Maximum number of messages about incorrect data in the log. #define MAX_ERROR_MESSAGES 10 diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 19ead09ae4b..372a2c2af07 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index d624550d233..0fa26ec5766 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 4bc3b2c8ba6..8778c1d6ad7 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index e5af154d7bd..6aeb17a5727 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 01ffd2b2e3d..fcec85cc1d8 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index 33b8e6ce8af..7d71e13a6e9 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -17,10 +17,12 @@ #include #include #include +#include #include #include + namespace po = boost::program_options; using ColumnType = uint32_t; From 6e8239d5d45934dbd090a4db5750bc947d500129 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 07:21:29 +0300 Subject: [PATCH 125/491] Add a test for #20315 --- .../01912_bad_cast_join_fuzz.reference | 10 ++++++++++ .../0_stateless/01912_bad_cast_join_fuzz.sql | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/queries/0_stateless/01912_bad_cast_join_fuzz.reference create mode 100644 tests/queries/0_stateless/01912_bad_cast_join_fuzz.sql diff --git a/tests/queries/0_stateless/01912_bad_cast_join_fuzz.reference b/tests/queries/0_stateless/01912_bad_cast_join_fuzz.reference new file mode 100644 index 00000000000..68707e5a4a7 --- /dev/null +++ b/tests/queries/0_stateless/01912_bad_cast_join_fuzz.reference @@ -0,0 +1,10 @@ +1023 0 \N +1024 1 \N +1025 2 \N +1026 3 \N +1027 4 \N +1028 5 \N +1029 6 \N +1030 7 \N +1031 8 \N +1032 9 \N diff --git a/tests/queries/0_stateless/01912_bad_cast_join_fuzz.sql b/tests/queries/0_stateless/01912_bad_cast_join_fuzz.sql new file mode 100644 index 00000000000..01e02a3be62 --- /dev/null +++ b/tests/queries/0_stateless/01912_bad_cast_join_fuzz.sql @@ -0,0 +1,16 @@ +SELECT + 1023 + l, + * +FROM +( + SELECT toLowCardinality(toNullable(number)) AS l + FROM system.numbers + LIMIT 10 +) AS s1 +ANY LEFT JOIN +( + SELECT toLowCardinality(toNullable(number)) AS r + FROM system.numbers + LIMIT 7 +) AS s2 ON (l + 1023) = (r * 3) +ORDER BY l, r; From f49463d1e5c1213428b31ca8bae58d33030e9e13 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 07:28:17 +0300 Subject: [PATCH 126/491] Fix clang-tidy --- src/Storages/MergeTree/MergeTreePartition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 894120e6179..bc951c2f3cf 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -34,7 +34,7 @@ namespace private: SipHash & hash; public: - LegacyFieldVisitorHash(SipHash & hash_) : hash(hash_) {} + explicit LegacyFieldVisitorHash(SipHash & hash_) : hash(hash_) {} void operator() (const Null &) const { From b4a9244978e393dd6568a1ba21e85eee0f40fd52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 14 Jun 2021 07:46:22 +0300 Subject: [PATCH 127/491] Fix style --- src/DataTypes/DataTypeAggregateFunction.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 904d4d2745e..f7ae3170119 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -1,5 +1,3 @@ -#include - #include #include From c08daa3ca52aec256b8e3dbb9d6c1739cd7bd231 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 8 Jun 2021 09:09:26 +0300 Subject: [PATCH 128/491] Detect linux version at runtime (for worked nested epoll) The #22109 adds the check but at compilation time, which is pointless, move the check into runtime. Remember nested epoll is required for async_socket_for_remote/use_hedged_requests, otherwise remote queries may stuck. --- src/Common/ErrorCodes.cpp | 1 + src/Common/VersionNumber.cpp | 61 +++++++++++++++++++++++ src/Common/VersionNumber.h | 54 ++++++++++++++++++++ src/Common/tests/gtest_version_number.cpp | 30 +++++++++++ src/Common/ya.make | 1 + src/Core/SettingsQuirks.cpp | 28 ++++++----- 6 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 src/Common/VersionNumber.cpp create mode 100644 src/Common/VersionNumber.h create mode 100644 src/Common/tests/gtest_version_number.cpp diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d840830bf28..d035a596bff 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -554,6 +554,7 @@ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ M(586, CANNOT_CREATE_FILE) \ + M(587, BAD_VERSION) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/VersionNumber.cpp b/src/Common/VersionNumber.cpp new file mode 100644 index 00000000000..8bf5b52ac0d --- /dev/null +++ b/src/Common/VersionNumber.cpp @@ -0,0 +1,61 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_VERSION; +} + +VersionNumber::VersionNumber(const std::vector & vec) +{ + if (vec.size() > SIZE) + throw Exception(ErrorCodes::BAD_VERSION, "Too much components ({})", vec.size()); + + if (vec.size() > 0) + std::get<0>(version) = vec[0]; + if (vec.size() > 1) + std::get<1>(version) = vec[1]; + if (vec.size() > 2) + std::get<2>(version) = vec[2]; +} + +std::string VersionNumber::toString() const +{ + return fmt::format("{}.{}.{}", + std::get<0>(version), std::get<1>(version), std::get<2>(version)); +} + +VersionNumber VersionNumber::fromString(std::string version, bool strict) +{ + if (version.empty()) + return VersionNumber{}; + + std::vector comp; + + char * start = &version.front(); + char * end = start; + const char * eos = &version.back() + 1; + + do + { + long value = strtol(start, &end, 10); + comp.push_back(value); + start = end + 1; + } + while (start < eos && (end < eos && *end == '.')); + + if (!strict && comp.size() > SIZE) + { + comp.resize(SIZE); + } + + return VersionNumber(std::move(comp)); +} + + +} diff --git a/src/Common/VersionNumber.h b/src/Common/VersionNumber.h new file mode 100644 index 00000000000..14c31f6da37 --- /dev/null +++ b/src/Common/VersionNumber.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +/// Simple numeric version representation. +/// +/// Supports only "major.minor.patch", other components are ignored. +struct VersionNumber +{ + explicit VersionNumber() = default; + + VersionNumber(const std::tuple & ver) + : version(ver) + {} + VersionNumber(const std::initializer_list & init) + : VersionNumber(std::vector(init)) + {} + VersionNumber(long major, long minor, long patch) + : version(major, minor, patch) + {} + + VersionNumber(const std::vector & vec); + + /// NOTE: operator<=> can be used once libc++ will be upgraded. + bool operator<(const VersionNumber & rhs) const { return version < rhs.version; } + bool operator<=(const VersionNumber & rhs) const { return version <= rhs.version; } + bool operator==(const VersionNumber & rhs) const { return version == rhs.version; } + bool operator>(const VersionNumber & rhs) const { return version > rhs.version; } + bool operator>=(const VersionNumber & rhs) const { return version >= rhs.version; } + + std::string toString() const; + + friend std::ostream & operator<<(std::ostream & os, const VersionNumber & v) + { + return os << v.toString(); + } + + /// @param strict - throws if number of components > 3 + static VersionNumber fromString(std::string version, bool strict); + +private: + using VersionTuple = std::tuple; + static constexpr size_t SIZE = std::tuple_size(); + + VersionTuple version{}; +}; + +} diff --git a/src/Common/tests/gtest_version_number.cpp b/src/Common/tests/gtest_version_number.cpp new file mode 100644 index 00000000000..58df945a600 --- /dev/null +++ b/src/Common/tests/gtest_version_number.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + +using namespace DB; + +TEST(VersionNumber, VersionNumber) +{ + VersionNumber version(1, 2, 3); + EXPECT_NE(VersionNumber(1, 1, 1), version); + EXPECT_EQ(VersionNumber(1, 2, 3), version); + EXPECT_GE(VersionNumber(1, 2, 3), version); + EXPECT_GT(VersionNumber(1, 2, 4), version); + EXPECT_LE(VersionNumber(1, 2, 3), version); + EXPECT_LT(VersionNumber(1, 2, 2), version); +} + +TEST(VersionNumber, fromString) +{ + EXPECT_EQ(VersionNumber::fromString("1.1.1", true), VersionNumber(1, 1, 1)); + EXPECT_EQ(VersionNumber::fromString("5.5.13prefix", true), VersionNumber(5, 5, 13)); + + EXPECT_THROW(VersionNumber::fromString("1.1.1.1", true), Exception); + EXPECT_NO_THROW(VersionNumber::fromString("1.1.1.1", false)); + + EXPECT_EQ(VersionNumber::fromString("1.1.1.1", false), VersionNumber(1, 1, 1)); + EXPECT_EQ(VersionNumber::fromString("1.1", true), VersionNumber(1, 1, 0)); + EXPECT_EQ(VersionNumber::fromString("1", true), VersionNumber(1, 0, 0)); + EXPECT_EQ(VersionNumber::fromString("", true), VersionNumber(0, 0, 0)); +} diff --git a/src/Common/ya.make b/src/Common/ya.make index 4f47eddd4f3..85a4a470338 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -86,6 +86,7 @@ SRCS( TraceCollector.cpp UTF8Helpers.cpp UnicodeBar.cpp + VersionNumber.cpp WeakHash.cpp ZooKeeper/IKeeper.cpp ZooKeeper/TestKeeper.cpp diff --git a/src/Core/SettingsQuirks.cpp b/src/Core/SettingsQuirks.cpp index 6d584fe8906..102b4a4cbc9 100644 --- a/src/Core/SettingsQuirks.cpp +++ b/src/Core/SettingsQuirks.cpp @@ -1,9 +1,10 @@ #include #include +#include +#include +#include #include - -#ifdef __linux__ -#include +#include /// Detect does epoll_wait with nested epoll fds works correctly. /// Polling nested epoll fds from epoll_wait is required for async_socket_for_remote and use_hedged_requests. @@ -14,19 +15,22 @@ /// [2]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=0c54a6a44bf3 bool nestedEpollWorks(Poco::Logger * log) { -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 13)) - /// the check is correct since there will be no more 5.5.x releases. + if (Poco::Environment::os() != POCO_OS_LINUX) + return true; + + DB::VersionNumber linux_version = DB::VersionNumber::fromString( + Poco::Environment::osVersion(), /* strict= */ false); + + /// the check is correct since there will be no more 5.5.x releases. + if (linux_version >= DB::VersionNumber{5, 5, 0} && linux_version < DB::VersionNumber{5, 6, 13}) + { if (log) LOG_WARNING(log, "Nested epoll_wait has some issues on kernels [5.5.0, 5.6.13). You should upgrade it to avoid possible issues."); return false; -#else - (void)log; - return true; -#endif + } + + return true; } -#else -bool nestedEpollWorks(Poco::Logger *) { return true; } -#endif namespace DB { From 5cc03535bf86bd04a1eed68a8fe17aa2ac4d6b42 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 9 Jun 2021 02:41:53 +0300 Subject: [PATCH 129/491] Set -Wno-covered-switch-default for unit_tests_dbms Otherwise code that uses EXPECT_EXIT() fails with -Werror v2: adjust only unit_tests_dbms, not googltest via contrib/CMakeListst.txt itself. --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 271287d46e8..4fa64e81f6c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -496,6 +496,7 @@ if (ENABLE_TESTS AND USE_GTEST) # gtest framework has substandard code target_compile_options(unit_tests_dbms PRIVATE -Wno-zero-as-null-pointer-constant + -Wno-covered-switch-default -Wno-undef -Wno-sign-compare -Wno-used-but-marked-unused From ba05bf397750c3e84c92520e63c3bb44e32bf486 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 8 Jun 2021 10:58:10 +0300 Subject: [PATCH 130/491] Use LOGICAL_ERROR instead of introducing BAD_VERSION But note, that tests with EXPECT_EXIT() does not work with --gtest_filter: a) only this test is included into unit_tests_dbms $ src/unit_tests_dbms --gtest_filter=VersionNumber* Running main() from ../contrib/googletest/googletest/src/gtest_main.cc Note: Google Test filter = VersionNumber* [==========] Running 2 tests from 1 test suite. [----------] Global test environment set-up. [----------] 2 tests from VersionNumber [ RUN ] VersionNumber.VersionNumber [ OK ] VersionNumber.VersionNumber (0 ms) [ RUN ] VersionNumber.fromString Segmentation fault (core dumped) (gdb) bt 0 0x00007fffda323d22 in raise () from /usr/lib/libc.so.6 1 0x00007fffda30d862 in abort () from /usr/lib/libc.so.6 2 0x00007fffdbe686f5 in DB::handle_error_code (msg=..., code=49, remote=false, trace=...) at Exception.cpp:49 3 0x00007fffdbe68846 in DB::Exception::Exception (this=0x7fffd5c5a240, msg=..., code=49, remote_=false) at Exception.cpp:60 4 0x00007fffe26b2cb3 in DB::Exception::Exception (this=0x7fffd5c5a240, code=49, fmt=..., args=@0x7fffffffc7e8: 4) at Exception.h:40 5 0x00007fffdbf4d201 in DB::VersionNumber::VersionNumber (this=0x7fffffffcc20, vec=...) at VersionNumber.cpp:17 6 0x00007fffdbf4d650 in DB::VersionNumber::fromString (version=..., strict=true) at VersionNumber.cpp:57 7 0x00005555555db53d in VersionNumber_fromString_Test::TestBody (this=0x7fffd5c12330) at gtest_version_number.cpp:24 b) regular build $ src/unit_tests_dbms --gtest_filter=VersionNumber* Running main() from ../contrib/googletest/googletest/src/gtest_main.cc Note: Google Test filter = VersionNumber* [==========] Running 2 tests from 1 test suite. [----------] Global test environment set-up. [----------] 2 tests from VersionNumber [ RUN ] VersionNumber.VersionNumber [ OK ] VersionNumber.VersionNumber (0 ms) [ RUN ] VersionNumber.fromString [ OK ] VersionNumber.fromString (495 ms) [----------] 2 tests from VersionNumber (495 ms total) [----------] Global test environment tear-down Segmentation fault (core dumped) (gdb) bt 0 testing::TestInfo::should_run (this=0xe0) at gtest.h:760 1 0x00007ffff7f6edd5 in testing::TestSuite::ShouldRunTest (test_info=0xe0) at gtest.h:1009 2 0x00007ffff7f6ebb2 in testing::internal::CountIf >, bool (*)(testing::TestInfo const*)> (c=..., predicate=0x7ffff7f6edc0 ) at gtest-internal-inl.h:294 3 0x00007ffff7f41ae5 in testing::TestSuite::test_to_run_count (this=0x7fffd5028500) at gtest.cc:2919 4 0x00007ffff7f41719 in testing::internal::SumOverTestSuiteList (case_list=..., method=(int (testing::TestSuite::*)(const testing::TestSuite * const)) 0x7ffff7f41ac0 ) at gtest.cc:377 5 0x00007ffff7f41ab9 in testing::internal::UnitTestImpl::test_to_run_count (this=0x7fffd5d5c000) at gtest.cc:1012 6 0x00007ffff7f4b59d in testing::UnitTest::test_to_run_count (this=0x7ffff7fbcb80 ) at gtest.cc:5059 7 0x00007ffff7f4c4ed in testing::internal::PrettyUnitTestResultPrinter::OnTestIterationEnd (this=0x7fffd5ca4a68, unit_test=...) at gtest.cc:3589 8 0x00007ffff7f4daf9 in testing::internal::TestEventRepeater::OnTestIterationEnd (this=0x7fffd5c83770, unit_test=..., iteration=0) at gtest.cc:3852 9 0x00007ffff7f57570 in testing::internal::UnitTestImpl::RunAllTests (this=0x7fffd5d5c000) at gtest.cc:5737 10 0x00007ffff7fa3a84 in testing::internal::HandleSehExceptionsInMethodIfSupported (object=0x7fffd5d5c000, method=(bool (testing::internal::UnitTestImpl::*)(testing::internal::UnitTestImpl * const)) 0x7ffff7f56f20 , location=0x7ffff7f11d9b "auxiliary test code (environments or event listeners)") at gtest.cc:2589 11 0x00007ffff7f71788 in testing::internal::HandleExceptionsInMethodIfSupported (object=0x7fffd5d5c000, method=(bool (testing::internal::UnitTestImpl::*)(testing::internal::UnitTestImpl * const)) 0x7ffff7f56f20 , location=0x7ffff7f11d9b "auxiliary test code (environments or event listeners)") at gtest.cc:2625 12 0x00007ffff7f56ea3 in testing::UnitTest::Run (this=0x7ffff7fbcb80 ) at gtest.cc:5291 --- src/Common/ErrorCodes.cpp | 1 - src/Common/VersionNumber.cpp | 4 ++-- src/Common/tests/gtest_version_number.cpp | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d035a596bff..d840830bf28 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -554,7 +554,6 @@ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ M(586, CANNOT_CREATE_FILE) \ - M(587, BAD_VERSION) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/VersionNumber.cpp b/src/Common/VersionNumber.cpp index 8bf5b52ac0d..16bf35a9083 100644 --- a/src/Common/VersionNumber.cpp +++ b/src/Common/VersionNumber.cpp @@ -8,13 +8,13 @@ namespace DB namespace ErrorCodes { - extern const int BAD_VERSION; + extern const int LOGICAL_ERROR; } VersionNumber::VersionNumber(const std::vector & vec) { if (vec.size() > SIZE) - throw Exception(ErrorCodes::BAD_VERSION, "Too much components ({})", vec.size()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Too much components ({})", vec.size()); if (vec.size() > 0) std::get<0>(version) = vec[0]; diff --git a/src/Common/tests/gtest_version_number.cpp b/src/Common/tests/gtest_version_number.cpp index 58df945a600..7a419e2ff31 100644 --- a/src/Common/tests/gtest_version_number.cpp +++ b/src/Common/tests/gtest_version_number.cpp @@ -20,7 +20,11 @@ TEST(VersionNumber, fromString) EXPECT_EQ(VersionNumber::fromString("1.1.1", true), VersionNumber(1, 1, 1)); EXPECT_EQ(VersionNumber::fromString("5.5.13prefix", true), VersionNumber(5, 5, 13)); +#ifdef ABORT_ON_LOGICAL_ERROR + EXPECT_EXIT(VersionNumber::fromString("1.1.1.1", true), testing::KilledBySignal(SIGABRT), ""); +#else EXPECT_THROW(VersionNumber::fromString("1.1.1.1", true), Exception); +#endif EXPECT_NO_THROW(VersionNumber::fromString("1.1.1.1", false)); EXPECT_EQ(VersionNumber::fromString("1.1.1.1", false), VersionNumber(1, 1, 1)); From 1b5ca07ff459a6192ef57e36a7a4114c5835be06 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 9 Jun 2021 02:40:32 +0300 Subject: [PATCH 131/491] Convert VersionNumber::fromString() to constructor --- src/Common/VersionNumber.cpp | 54 +++++++++++------------ src/Common/VersionNumber.h | 7 +-- src/Common/tests/gtest_version_number.cpp | 18 ++++---- src/Core/SettingsQuirks.cpp | 3 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/Common/VersionNumber.cpp b/src/Common/VersionNumber.cpp index 16bf35a9083..60ea850eace 100644 --- a/src/Common/VersionNumber.cpp +++ b/src/Common/VersionNumber.cpp @@ -11,6 +11,33 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +VersionNumber::VersionNumber(std::string version_string, bool strict) +{ + if (version_string.empty()) + return; + + std::vector comp; + + char * start = &version_string.front(); + char * end = start; + const char * eos = &version_string.back() + 1; + + do + { + long value = strtol(start, &end, 10); + comp.push_back(value); + start = end + 1; + } + while (start < eos && (end < eos && *end == '.')); + + if (!strict && comp.size() > SIZE) + { + comp.resize(SIZE); + } + + *this = comp; +} + VersionNumber::VersionNumber(const std::vector & vec) { if (vec.size() > SIZE) @@ -30,32 +57,5 @@ std::string VersionNumber::toString() const std::get<0>(version), std::get<1>(version), std::get<2>(version)); } -VersionNumber VersionNumber::fromString(std::string version, bool strict) -{ - if (version.empty()) - return VersionNumber{}; - - std::vector comp; - - char * start = &version.front(); - char * end = start; - const char * eos = &version.back() + 1; - - do - { - long value = strtol(start, &end, 10); - comp.push_back(value); - start = end + 1; - } - while (start < eos && (end < eos && *end == '.')); - - if (!strict && comp.size() > SIZE) - { - comp.resize(SIZE); - } - - return VersionNumber(std::move(comp)); -} - } diff --git a/src/Common/VersionNumber.h b/src/Common/VersionNumber.h index 14c31f6da37..28bdd35d7db 100644 --- a/src/Common/VersionNumber.h +++ b/src/Common/VersionNumber.h @@ -24,6 +24,10 @@ struct VersionNumber VersionNumber(long major, long minor, long patch) : version(major, minor, patch) {} + /// Parse version number from string. + /// + /// @param strict - throws if number of components > 3 + VersionNumber(std::string version, bool strict); VersionNumber(const std::vector & vec); @@ -41,9 +45,6 @@ struct VersionNumber return os << v.toString(); } - /// @param strict - throws if number of components > 3 - static VersionNumber fromString(std::string version, bool strict); - private: using VersionTuple = std::tuple; static constexpr size_t SIZE = std::tuple_size(); diff --git a/src/Common/tests/gtest_version_number.cpp b/src/Common/tests/gtest_version_number.cpp index 7a419e2ff31..055b2cc336b 100644 --- a/src/Common/tests/gtest_version_number.cpp +++ b/src/Common/tests/gtest_version_number.cpp @@ -17,18 +17,18 @@ TEST(VersionNumber, VersionNumber) TEST(VersionNumber, fromString) { - EXPECT_EQ(VersionNumber::fromString("1.1.1", true), VersionNumber(1, 1, 1)); - EXPECT_EQ(VersionNumber::fromString("5.5.13prefix", true), VersionNumber(5, 5, 13)); + EXPECT_EQ(VersionNumber("1.1.1", true), VersionNumber(1, 1, 1)); + EXPECT_EQ(VersionNumber("5.5.13prefix", true), VersionNumber(5, 5, 13)); #ifdef ABORT_ON_LOGICAL_ERROR - EXPECT_EXIT(VersionNumber::fromString("1.1.1.1", true), testing::KilledBySignal(SIGABRT), ""); + EXPECT_EXIT(VersionNumber("1.1.1.1", true), testing::KilledBySignal(SIGABRT), ""); #else - EXPECT_THROW(VersionNumber::fromString("1.1.1.1", true), Exception); + EXPECT_THROW(VersionNumber("1.1.1.1", true), Exception); #endif - EXPECT_NO_THROW(VersionNumber::fromString("1.1.1.1", false)); + EXPECT_NO_THROW(VersionNumber("1.1.1.1", false)); - EXPECT_EQ(VersionNumber::fromString("1.1.1.1", false), VersionNumber(1, 1, 1)); - EXPECT_EQ(VersionNumber::fromString("1.1", true), VersionNumber(1, 1, 0)); - EXPECT_EQ(VersionNumber::fromString("1", true), VersionNumber(1, 0, 0)); - EXPECT_EQ(VersionNumber::fromString("", true), VersionNumber(0, 0, 0)); + EXPECT_EQ(VersionNumber("1.1.1.1", false), VersionNumber(1, 1, 1)); + EXPECT_EQ(VersionNumber("1.1", true), VersionNumber(1, 1, 0)); + EXPECT_EQ(VersionNumber("1", true), VersionNumber(1, 0, 0)); + EXPECT_EQ(VersionNumber("", true), VersionNumber(0, 0, 0)); } diff --git a/src/Core/SettingsQuirks.cpp b/src/Core/SettingsQuirks.cpp index 102b4a4cbc9..45ab3ca1404 100644 --- a/src/Core/SettingsQuirks.cpp +++ b/src/Core/SettingsQuirks.cpp @@ -18,8 +18,7 @@ bool nestedEpollWorks(Poco::Logger * log) if (Poco::Environment::os() != POCO_OS_LINUX) return true; - DB::VersionNumber linux_version = DB::VersionNumber::fromString( - Poco::Environment::osVersion(), /* strict= */ false); + DB::VersionNumber linux_version(Poco::Environment::osVersion(), /* strict= */ false); /// the check is correct since there will be no more 5.5.x releases. if (linux_version >= DB::VersionNumber{5, 5, 0} && linux_version < DB::VersionNumber{5, 6, 13}) From 055e9d87f06e9682bb6c63b6837bd3db4e587f17 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 9 Jun 2021 02:40:39 +0300 Subject: [PATCH 132/491] Use VersionNumber for detecting renameat2() --- src/Common/renameat2.cpp | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 26d90427889..2615445b482 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #if defined(linux) || defined(__linux) || defined(__linux__) @@ -7,7 +9,6 @@ #include #include #include -#include #endif namespace fs = std::filesystem; @@ -27,22 +28,9 @@ namespace ErrorCodes static bool supportsRenameat2Impl() { #if defined(__NR_renameat2) - /// renameat2 is available in linux since 3.15 - struct utsname sysinfo; - if (uname(&sysinfo)) - return false; - char * point = nullptr; - auto v_major = strtol(sysinfo.release, &point, 10); - - errno = 0; - if (errno || *point != '.' || v_major < 3) - return false; - if (3 < v_major) - return true; - - errno = 0; - auto v_minor = strtol(point + 1, nullptr, 10); - return !errno && 15 <= v_minor; + DB::VersionNumber renameat2_minimal_version(3, 15, 0); + DB::VersionNumber linux_version(Poco::Environment::osVersion(), /* strict= */ false); + return linux_version >= renameat2_minimal_version; #else return false; #endif From b5a2fad99366e932215caa3aa56fc07457c77c45 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 14 Jun 2021 10:23:29 +0300 Subject: [PATCH 133/491] More generic VersionNumber --- src/Common/VersionNumber.cpp | 68 ++++++++++++----------- src/Common/VersionNumber.h | 38 ++++++------- src/Common/renameat2.cpp | 2 +- src/Common/tests/gtest_version_number.cpp | 20 ++----- src/Core/SettingsQuirks.cpp | 2 +- 5 files changed, 61 insertions(+), 69 deletions(-) diff --git a/src/Common/VersionNumber.cpp b/src/Common/VersionNumber.cpp index 60ea850eace..08dcdd379bc 100644 --- a/src/Common/VersionNumber.cpp +++ b/src/Common/VersionNumber.cpp @@ -1,23 +1,15 @@ #include -#include #include #include namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -VersionNumber::VersionNumber(std::string version_string, bool strict) +VersionNumber::VersionNumber(std::string version_string) { if (version_string.empty()) return; - std::vector comp; - char * start = &version_string.front(); char * end = start; const char * eos = &version_string.back() + 1; @@ -25,37 +17,49 @@ VersionNumber::VersionNumber(std::string version_string, bool strict) do { long value = strtol(start, &end, 10); - comp.push_back(value); + components.push_back(value); start = end + 1; } while (start < eos && (end < eos && *end == '.')); - - if (!strict && comp.size() > SIZE) - { - comp.resize(SIZE); - } - - *this = comp; -} - -VersionNumber::VersionNumber(const std::vector & vec) -{ - if (vec.size() > SIZE) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Too much components ({})", vec.size()); - - if (vec.size() > 0) - std::get<0>(version) = vec[0]; - if (vec.size() > 1) - std::get<1>(version) = vec[1]; - if (vec.size() > 2) - std::get<2>(version) = vec[2]; } std::string VersionNumber::toString() const { - return fmt::format("{}.{}.{}", - std::get<0>(version), std::get<1>(version), std::get<2>(version)); + std::string str; + for (long v : components) + { + if (!str.empty()) + str += '.'; + str += std::to_string(v); + } + return str; } +int VersionNumber::compare(const VersionNumber & rhs) const +{ + size_t min = std::min(components.size(), rhs.components.size()); + for (size_t i = 0; i < min; ++i) + { + if (int d = components[i] - rhs.components[i]) + return d; + } + + if (components.size() > min) + { + if (components[min] != 0) + return components[min]; + else + return 1; + } + else if (rhs.components.size() > min) + { + if (rhs.components[min] != 0) + return -rhs.components[min]; + else + return -1; + } + + return 0; +} } diff --git a/src/Common/VersionNumber.h b/src/Common/VersionNumber.h index 28bdd35d7db..0234c14a44c 100644 --- a/src/Common/VersionNumber.h +++ b/src/Common/VersionNumber.h @@ -10,46 +10,42 @@ namespace DB /// Simple numeric version representation. /// -/// Supports only "major.minor.patch", other components are ignored. +/// Based on QVersionNumber. struct VersionNumber { explicit VersionNumber() = default; - VersionNumber(const std::tuple & ver) - : version(ver) - {} VersionNumber(const std::initializer_list & init) - : VersionNumber(std::vector(init)) + : components(init) {} - VersionNumber(long major, long minor, long patch) - : version(major, minor, patch) + VersionNumber(long major, long minor = 0, long patch = 0) + : components{major, minor, patch} + {} + VersionNumber(const std::vector & components_) + : components(components_) {} - /// Parse version number from string. - /// - /// @param strict - throws if number of components > 3 - VersionNumber(std::string version, bool strict); - VersionNumber(const std::vector & vec); + /// Parse version number from string. + VersionNumber(std::string version); /// NOTE: operator<=> can be used once libc++ will be upgraded. - bool operator<(const VersionNumber & rhs) const { return version < rhs.version; } - bool operator<=(const VersionNumber & rhs) const { return version <= rhs.version; } - bool operator==(const VersionNumber & rhs) const { return version == rhs.version; } - bool operator>(const VersionNumber & rhs) const { return version > rhs.version; } - bool operator>=(const VersionNumber & rhs) const { return version >= rhs.version; } + bool operator<(const VersionNumber & rhs) const { return compare(rhs.components) < 0; } + bool operator<=(const VersionNumber & rhs) const { return compare(rhs.components) <= 0; } + bool operator==(const VersionNumber & rhs) const { return compare(rhs.components) == 0; } + bool operator>(const VersionNumber & rhs) const { return compare(rhs.components) > 0; } + bool operator>=(const VersionNumber & rhs) const { return compare(rhs.components) >= 0; } std::string toString() const; - friend std::ostream & operator<<(std::ostream & os, const VersionNumber & v) { return os << v.toString(); } private: - using VersionTuple = std::tuple; - static constexpr size_t SIZE = std::tuple_size(); + using Components = std::vector; + Components components; - VersionTuple version{}; + int compare(const VersionNumber & rhs) const; }; } diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index 2615445b482..d89fdf19204 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -29,7 +29,7 @@ static bool supportsRenameat2Impl() { #if defined(__NR_renameat2) DB::VersionNumber renameat2_minimal_version(3, 15, 0); - DB::VersionNumber linux_version(Poco::Environment::osVersion(), /* strict= */ false); + DB::VersionNumber linux_version(Poco::Environment::osVersion()); return linux_version >= renameat2_minimal_version; #else return false; diff --git a/src/Common/tests/gtest_version_number.cpp b/src/Common/tests/gtest_version_number.cpp index 055b2cc336b..c287795b20e 100644 --- a/src/Common/tests/gtest_version_number.cpp +++ b/src/Common/tests/gtest_version_number.cpp @@ -1,5 +1,4 @@ #include -#include #include using namespace DB; @@ -17,18 +16,11 @@ TEST(VersionNumber, VersionNumber) TEST(VersionNumber, fromString) { - EXPECT_EQ(VersionNumber("1.1.1", true), VersionNumber(1, 1, 1)); - EXPECT_EQ(VersionNumber("5.5.13prefix", true), VersionNumber(5, 5, 13)); + EXPECT_EQ(VersionNumber("1.1.1"), VersionNumber(1, 1, 1)); + EXPECT_EQ(VersionNumber("5.5.13prefix"), VersionNumber(5, 5, 13)); -#ifdef ABORT_ON_LOGICAL_ERROR - EXPECT_EXIT(VersionNumber("1.1.1.1", true), testing::KilledBySignal(SIGABRT), ""); -#else - EXPECT_THROW(VersionNumber("1.1.1.1", true), Exception); -#endif - EXPECT_NO_THROW(VersionNumber("1.1.1.1", false)); - - EXPECT_EQ(VersionNumber("1.1.1.1", false), VersionNumber(1, 1, 1)); - EXPECT_EQ(VersionNumber("1.1", true), VersionNumber(1, 1, 0)); - EXPECT_EQ(VersionNumber("1", true), VersionNumber(1, 0, 0)); - EXPECT_EQ(VersionNumber("", true), VersionNumber(0, 0, 0)); + EXPECT_GT(VersionNumber("1.1.1.1"), VersionNumber(1, 1, 1)); + EXPECT_LT(VersionNumber("1.1"), VersionNumber(1, 1, 0)); + EXPECT_LT(VersionNumber("1"), VersionNumber(1, 0, 0)); + EXPECT_LT(VersionNumber(""), VersionNumber(0, 0, 0)); } diff --git a/src/Core/SettingsQuirks.cpp b/src/Core/SettingsQuirks.cpp index 45ab3ca1404..f969c42680c 100644 --- a/src/Core/SettingsQuirks.cpp +++ b/src/Core/SettingsQuirks.cpp @@ -18,7 +18,7 @@ bool nestedEpollWorks(Poco::Logger * log) if (Poco::Environment::os() != POCO_OS_LINUX) return true; - DB::VersionNumber linux_version(Poco::Environment::osVersion(), /* strict= */ false); + DB::VersionNumber linux_version(Poco::Environment::osVersion()); /// the check is correct since there will be no more 5.5.x releases. if (linux_version >= DB::VersionNumber{5, 5, 0} && linux_version < DB::VersionNumber{5, 6, 13}) From 40abdaf24a4b3915e87fcb73736bbab23bf117b9 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 14 Jun 2021 10:36:02 +0300 Subject: [PATCH 134/491] Drop unnecessary qualified namespace --- src/Common/renameat2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/renameat2.cpp b/src/Common/renameat2.cpp index d89fdf19204..78b088d5eb9 100644 --- a/src/Common/renameat2.cpp +++ b/src/Common/renameat2.cpp @@ -28,8 +28,8 @@ namespace ErrorCodes static bool supportsRenameat2Impl() { #if defined(__NR_renameat2) - DB::VersionNumber renameat2_minimal_version(3, 15, 0); - DB::VersionNumber linux_version(Poco::Environment::osVersion()); + VersionNumber renameat2_minimal_version(3, 15, 0); + VersionNumber linux_version(Poco::Environment::osVersion()); return linux_version >= renameat2_minimal_version; #else return false; From 92765a359523bda768ce766a74df473defcb6107 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 14 Jun 2021 10:41:36 +0300 Subject: [PATCH 135/491] Fix clang-tidy warnings in VersionNumber (use Int64 over long) --- src/Common/VersionNumber.cpp | 4 ++-- src/Common/VersionNumber.h | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Common/VersionNumber.cpp b/src/Common/VersionNumber.cpp index 08dcdd379bc..b25fd994947 100644 --- a/src/Common/VersionNumber.cpp +++ b/src/Common/VersionNumber.cpp @@ -16,7 +16,7 @@ VersionNumber::VersionNumber(std::string version_string) do { - long value = strtol(start, &end, 10); + Int64 value = strtol(start, &end, 10); components.push_back(value); start = end + 1; } @@ -26,7 +26,7 @@ VersionNumber::VersionNumber(std::string version_string) std::string VersionNumber::toString() const { std::string str; - for (long v : components) + for (Int64 v : components) { if (!str.empty()) str += '.'; diff --git a/src/Common/VersionNumber.h b/src/Common/VersionNumber.h index 0234c14a44c..fb5de7fdfb6 100644 --- a/src/Common/VersionNumber.h +++ b/src/Common/VersionNumber.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -15,13 +16,13 @@ struct VersionNumber { explicit VersionNumber() = default; - VersionNumber(const std::initializer_list & init) + VersionNumber(const std::initializer_list & init) : components(init) {} - VersionNumber(long major, long minor = 0, long patch = 0) + VersionNumber(Int64 major, Int64 minor = 0, Int64 patch = 0) : components{major, minor, patch} {} - VersionNumber(const std::vector & components_) + VersionNumber(const std::vector & components_) : components(components_) {} @@ -42,7 +43,7 @@ struct VersionNumber } private: - using Components = std::vector; + using Components = std::vector; Components components; int compare(const VersionNumber & rhs) const; From e8e7682a7df859b7a7533b87597a3a45664662bb Mon Sep 17 00:00:00 2001 From: "Matwey V. Kornilov" Date: Mon, 14 Jun 2021 12:43:16 +0300 Subject: [PATCH 136/491] Add missed #include std::optional is used for user_dn_detection; Signed-off-by: Matwey V. Kornilov --- src/Access/LDAPClient.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index 388e7ad0f0d..85003da31f1 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -14,6 +14,7 @@ #endif #include +#include #include #include From 414bde9f2a876733e9baeab7b9cdcb6e6601c675 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Mon, 14 Jun 2021 15:39:53 +0300 Subject: [PATCH 137/491] edited after review and translated notes in RU examples --- .../parametric-functions.md | 10 +- .../parametric-functions.md | 110 +++++++++--------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index e0953f5fa28..7a4e55f00cb 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -528,8 +528,8 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event - `base` — Used to set the base point. - head — Set the base point to the first event. - tail — Set the base point to the last event. - - first_match — Set the base point to the first matched event1. - - last_match — Set the base point to the last matched event1. + - first_match — Set the base point to the first matched `event1`. + - last_match — Set the base point to the last matched `event1`. **Arguments** @@ -547,7 +547,7 @@ Type: [Nullable(String)](../../sql-reference/data-types/nullable.md). **Example** -It can be used when events are A->B->C->E->F and you want to know the event following B->C, which is E. +It can be used when events are A->B->C->D->E-> and you want to know the event following B->C, which is D. The query statement searching the event following A->B: @@ -560,7 +560,7 @@ ENGINE = MergeTree() PARTITION BY toYYYYMMDD(dt) ORDER BY id; -INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'E') (5, 1, 'F'); +INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'D') (5, 1, 'E'); SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'A', page = 'A', page = 'B') as next_flow FROM test_flow GROUP BY id; ``` @@ -640,7 +640,7 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p 1970-01-01 09:00:04 2 Basket The result 1970-01-01 09:00:01 3 Gift // Base point -1970-01-01 09:00:02 3 Home // Thre result +1970-01-01 09:00:02 3 Home // The result 1970-01-01 09:00:03 3 Gift 1970-01-01 09:00:04 3 Basket ``` diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index bb0e951a2f1..dc16d139319 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -517,8 +517,8 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event - `base` — используется для задания начальной точки. - head — установить начальную точку на первое событие цепочки. - tail — установить начальную точку на последнее событие цепочки. - - first_match — установить начальную точку на первое соответствующее событие1. - - last_match — установить начальную точку на последнее соответствующее событие1. + - first_match — установить начальную точку на первое соответствующее событие `event1`. + - last_match — установить начальную точку на последнее соответствующее событие `event1`. **Аргументы** @@ -529,16 +529,16 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event **Возвращаемые значения** -- `event_column[next_index]` — если шаблон совпал и существует следующее значение. -- `NULL` — если шаблон не совпал или следующее значение не существует. +- `event_column[next_index]` — если есть совпадение с шаблоном и существует следующее значение. +- `NULL` — если нет совпадений с шаблоном или следующего значения не существует. Тип: [Nullable(String)](../../sql-reference/data-types/nullable.md). **Пример** -Функцию можно использовать, если есть цепочка событий A-> B-> C-> E-> F, и вы хотите определить событие, следующее за B-> C, то есть E. +Функцию можно использовать, если есть цепочка событий A->B->C->D->E, и вы хотите определить событие, следующее за B->C, то есть D. -Оператор запроса ищет событие после A-> B: +Оператор запроса ищет событие после A->B: ``` sql CREATE TABLE test_flow ( @@ -549,7 +549,7 @@ ENGINE = MergeTree() PARTITION BY toYYYYMMDD(dt) ORDER BY id; -INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'E') (5, 1, 'F'); +INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'D') (5, 1, 'E'); SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'A', page = 'A', page = 'B') as next_flow FROM test_flow GROUP BY id; ``` @@ -576,16 +576,16 @@ INSERT INTO test_flow VALUES (1, 3, 'Gift') (2, 3, 'Home') (3, 3, 'Gift') (4, 3, SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'Home', page = 'Home', page = 'Gift') FROM test_flow GROUP BY id; dt id page - 1970-01-01 09:00:01 1 Home // Base point, Matched with Home - 1970-01-01 09:00:02 1 Gift // Matched with Gift - 1970-01-01 09:00:03 1 Exit // The result + 1970-01-01 09:00:01 1 Home // Исходная точка, совпадение с Home + 1970-01-01 09:00:02 1 Gift // Совпадение с Gift + 1970-01-01 09:00:03 1 Exit // Результат - 1970-01-01 09:00:01 2 Home // Base point, Matched with Home - 1970-01-01 09:00:02 2 Home // Unmatched with Gift + 1970-01-01 09:00:01 2 Home // Исходная точка, совпадение с Home + 1970-01-01 09:00:02 2 Home // Несовпадение с Gift 1970-01-01 09:00:03 2 Gift 1970-01-01 09:00:04 2 Basket - 1970-01-01 09:00:01 3 Gift // Base point, Unmatched with Home + 1970-01-01 09:00:01 3 Gift // Исходная точка, несовпадение с Home 1970-01-01 09:00:02 3 Home 1970-01-01 09:00:03 3 Gift 1970-01-01 09:00:04 3 Basket @@ -599,17 +599,17 @@ SELECT id, sequenceNextNode('backward', 'tail')(dt, page, page = 'Basket', page dt id page 1970-01-01 09:00:01 1 Home 1970-01-01 09:00:02 1 Gift -1970-01-01 09:00:03 1 Exit // Base point, Unmatched with Basket +1970-01-01 09:00:03 1 Exit // Исходная точка, несовпадение с Basket 1970-01-01 09:00:01 2 Home -1970-01-01 09:00:02 2 Home // The result -1970-01-01 09:00:03 2 Gift // Matched with Gift -1970-01-01 09:00:04 2 Basket // Base point, Matched with Basket +1970-01-01 09:00:02 2 Home // Результат +1970-01-01 09:00:03 2 Gift // Совпадение с Gift +1970-01-01 09:00:04 2 Basket // Исходная точка, совпадение с Basket 1970-01-01 09:00:01 3 Gift -1970-01-01 09:00:02 3 Home // The result -1970-01-01 09:00:03 3 Gift // Base point, Matched with Gift -1970-01-01 09:00:04 3 Basket // Base point, Matched with Basket +1970-01-01 09:00:02 3 Home // Результат +1970-01-01 09:00:03 3 Gift // Исходная точка, совпадение с Gift +1970-01-01 09:00:04 3 Basket // Исходная точка, совпадение с Basket ``` @@ -620,16 +620,16 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p dt id page 1970-01-01 09:00:01 1 Home -1970-01-01 09:00:02 1 Gift // Base point -1970-01-01 09:00:03 1 Exit // The result +1970-01-01 09:00:02 1 Gift // Исходная точка +1970-01-01 09:00:03 1 Exit // Результат 1970-01-01 09:00:01 2 Home 1970-01-01 09:00:02 2 Home -1970-01-01 09:00:03 2 Gift // Base point -1970-01-01 09:00:04 2 Basket The result +1970-01-01 09:00:03 2 Gift // Исходная точка +1970-01-01 09:00:04 2 Basket Результат -1970-01-01 09:00:01 3 Gift // Base point -1970-01-01 09:00:02 3 Home // Thre result +1970-01-01 09:00:01 3 Gift // Исходная точка +1970-01-01 09:00:02 3 Home // Результат 1970-01-01 09:00:03 3 Gift 1970-01-01 09:00:04 3 Basket ``` @@ -639,17 +639,17 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p dt id page 1970-01-01 09:00:01 1 Home -1970-01-01 09:00:02 1 Gift // Base point -1970-01-01 09:00:03 1 Exit // Unmatched with Home +1970-01-01 09:00:02 1 Gift // Исходная точка +1970-01-01 09:00:03 1 Exit // Несовпадение с Home 1970-01-01 09:00:01 2 Home 1970-01-01 09:00:02 2 Home -1970-01-01 09:00:03 2 Gift // Base point -1970-01-01 09:00:04 2 Basket // Unmatched with Home +1970-01-01 09:00:03 2 Gift // Исходная точка +1970-01-01 09:00:04 2 Basket // Несовпадение с Home -1970-01-01 09:00:01 3 Gift // Base point -1970-01-01 09:00:02 3 Home // Matched with Home -1970-01-01 09:00:03 3 Gift // The result +1970-01-01 09:00:01 3 Gift // Исходная точка +1970-01-01 09:00:02 3 Home // Совпадение с Home +1970-01-01 09:00:03 3 Gift // Результат 1970-01-01 09:00:04 3 Basket ``` @@ -660,18 +660,18 @@ SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', p SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; dt id page -1970-01-01 09:00:01 1 Home // The result -1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:01 1 Home // Результат +1970-01-01 09:00:02 1 Gift // Исходная точка 1970-01-01 09:00:03 1 Exit 1970-01-01 09:00:01 2 Home -1970-01-01 09:00:02 2 Home // The result -1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:02 2 Home // Результат +1970-01-01 09:00:03 2 Gift // Исходная точка 1970-01-01 09:00:04 2 Basket 1970-01-01 09:00:01 3 Gift -1970-01-01 09:00:02 3 Home // The result -1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:02 3 Home // Результат +1970-01-01 09:00:03 3 Gift // Исходная точка 1970-01-01 09:00:04 3 Basket ``` @@ -679,18 +679,18 @@ SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', p SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; dt id page -1970-01-01 09:00:01 1 Home // Matched with Home, the result is null -1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:01 1 Home // Совпадение с Home, результат `Null` +1970-01-01 09:00:02 1 Gift // Исходная точка 1970-01-01 09:00:03 1 Exit -1970-01-01 09:00:01 2 Home // The result -1970-01-01 09:00:02 2 Home // Matched with Home -1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:01 2 Home // Результат +1970-01-01 09:00:02 2 Home // Совпадение с Home +1970-01-01 09:00:03 2 Gift // Исходная точка 1970-01-01 09:00:04 2 Basket -1970-01-01 09:00:01 3 Gift // The result -1970-01-01 09:00:02 3 Home // Matched with Home -1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:01 3 Gift // Результат +1970-01-01 09:00:02 3 Home // Совпадение с Home +1970-01-01 09:00:03 3 Gift // Исходная точка 1970-01-01 09:00:04 3 Basket ``` @@ -716,7 +716,7 @@ INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3 SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref - 1970-01-01 09:00:01 1 A ref4 // The head can not be base point because the ref column of the head unmatched with 'ref1'. + 1970-01-01 09:00:01 1 A ref4 // Начало не может быть исходной точкой, поскольку столбец ref не соответствует 'ref1'. 1970-01-01 09:00:02 1 A ref3 1970-01-01 09:00:03 1 B ref2 1970-01-01 09:00:04 1 B ref1 @@ -729,16 +729,16 @@ SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = ' 1970-01-01 09:00:01 1 A ref4 1970-01-01 09:00:02 1 A ref3 1970-01-01 09:00:03 1 B ref2 - 1970-01-01 09:00:04 1 B ref1 // The tail can not be base point because the ref column of the tail unmatched with 'ref4'. + 1970-01-01 09:00:04 1 B ref1 // Конец не может быть исходной точкой, поскольку столбец ref не соответствует 'ref4'. ``` ``` sql SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', page = 'A') FROM test_flow_basecond GROUP BY id; dt id page ref - 1970-01-01 09:00:01 1 A ref4 // This row can not be base point because the ref column unmatched with 'ref3'. - 1970-01-01 09:00:02 1 A ref3 // Base point - 1970-01-01 09:00:03 1 B ref2 // The result + 1970-01-01 09:00:01 1 A ref4 // Эта строка не может быть исходной точкой, поскольку столбец ref не соответствует 'ref3'. + 1970-01-01 09:00:02 1 A ref3 // Исходная точка + 1970-01-01 09:00:03 1 B ref2 // Результат 1970-01-01 09:00:04 1 B ref1 ``` @@ -747,7 +747,7 @@ SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, ref = 'ref2', pa dt id page ref 1970-01-01 09:00:01 1 A ref4 - 1970-01-01 09:00:02 1 A ref3 // The result - 1970-01-01 09:00:03 1 B ref2 // Base point - 1970-01-01 09:00:04 1 B ref1 // This row can not be base point because the ref column unmatched with 'ref2'. + 1970-01-01 09:00:02 1 A ref3 // Результат + 1970-01-01 09:00:03 1 B ref2 // Исходная точка + 1970-01-01 09:00:04 1 B ref1 // Эта строка не может быть исходной точкой, поскольку столбец ref не соответствует 'ref2'. ``` From 9e11daa8c64185fd497b80c725a05f3e14c5a70f Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Mon, 14 Jun 2021 15:54:27 +0300 Subject: [PATCH 138/491] minor changes --- .../sql-reference/aggregate-functions/parametric-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 7a4e55f00cb..33df62e5c61 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -547,7 +547,7 @@ Type: [Nullable(String)](../../sql-reference/data-types/nullable.md). **Example** -It can be used when events are A->B->C->D->E-> and you want to know the event following B->C, which is D. +It can be used when events are A->B->C->D->E and you want to know the event following B->C, which is D. The query statement searching the event following A->B: From ee7169e547a47b89dce61788622b95759896ca12 Mon Sep 17 00:00:00 2001 From: Nickita Date: Tue, 4 May 2021 16:12:39 +0300 Subject: [PATCH 139/491] fix typos in docs --- docs/en/development/architecture.md | 4 ++-- docs/en/engines/table-engines/mergetree-family/mergetree.md | 6 +++--- .../table-engines/mergetree-family/summingmergetree.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 424052001fd..e2e40603d30 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -112,7 +112,7 @@ A hand-written recursive descent parser parses a query. For example, `ParserSele Interpreters are responsible for creating the query execution pipeline from an `AST`. There are simple interpreters, such as `InterpreterExistsQuery` and `InterpreterDropQuery`, or the more sophisticated `InterpreterSelectQuery`. The query execution pipeline is a combination of block input or output streams. For example, the result of interpreting the `SELECT` query is the `IBlockInputStream` to read the result set from; the result of the INSERT query is the `IBlockOutputStream` to write data for insertion to, and the result of interpreting the `INSERT SELECT` query is the `IBlockInputStream` that returns an empty result set on the first read, but that copies data from `SELECT` to `INSERT` at the same time. -`InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations or query. +`InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are done. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted to separate classes to allow modular transformations of query. ## Functions {#functions} @@ -169,7 +169,7 @@ There is no global query plan for distributed query execution. Each node has its `MergeTree` is a family of storage engines that supports indexing by primary key. The primary key can be an arbitrary tuple of columns or expressions. Data in a `MergeTree` table is stored in “parts”. Each part stores data in the primary key order, so data is ordered lexicographically by the primary key tuple. All the table columns are stored in separate `column.bin` files in these parts. The files consist of compressed blocks. Each block is usually from 64 KB to 1 MB of uncompressed data, depending on the average value size. The blocks consist of column values placed contiguously one after the other. Column values are in the same order for each column (the primary key defines the order), so when you iterate by many columns, you get values for the corresponding rows. -The primary key itself is “sparse”. It does not address every single row, but only some ranges of data. A separate `primary.idx` file has the value of the primary key for each N-th row, where N is called `index_granularity` (usually, N = 8192). Also, for each column, we have `column.mrk` files with “marks,” which are offsets to each N-th row in the data file. Each mark is a pair: the offset in the file to the beginning of the compressed block, and the offset in the decompressed block to the beginning of data. Usually, compressed blocks are aligned by marks, and the offset in the decompressed block is zero. Data for `primary.idx` always resides in memory, and data for `column.mrk` files is cached. +The primary key itself is “sparse”. It does not address every single row, but only some ranges of data. A separate `primary.idx` file has the value of the primary key for each N-th row, where N is called `index_granularity` (usually, N = 8192). Also, for each column, we have `column.mrk` files with “marks”, which are offsets to each N-th row in the data file. Each mark is a pair: the offset in the file to the beginning of the compressed block, and the offset in the decompressed block to the beginning of data. Usually, compressed blocks are aligned by marks, and the offset in the decompressed block is zero. Data for `primary.idx` always resides in memory, and data for `column.mrk` files is cached. When we are going to read something from a part in `MergeTree`, we look at `primary.idx` data and locate ranges that could contain requested data, then look at `column.mrk` data and calculate offsets for where to start reading those ranges. Because of sparseness, excess data may be read. ClickHouse is not suitable for a high load of simple point queries, because the entire range with `index_granularity` rows must be read for each key, and the entire compressed block must be decompressed for each column. We made the index sparse because we must be able to maintain trillions of rows per single server without noticeable memory consumption for the index. Also, because the primary key is sparse, it is not unique: it cannot check the existence of the key in the table at INSERT time. You could have many rows with the same key in a table. diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index eb87dbc6580..1c5d7005ae8 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -17,7 +17,7 @@ Main features: - Partitions can be used if the [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) is specified. - ClickHouse supports certain operations with partitions that are more effective than general operations on the same data with the same result. ClickHouse also automatically cuts off the partition data where the partitioning key is specified in the query. + ClickHouse supports certain operations with partitions that are more efficient than general operations on the same data with the same result. ClickHouse also automatically cuts off the partition data where the partitioning key is specified in the query. - Data replication support. @@ -83,7 +83,7 @@ For a description of parameters, see the [CREATE query description](../../../sql Expression must have one `Date` or `DateTime` column as a result. Example: `TTL date + INTERVAL 1 DAY` - Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can specified, but there should be no more than one `DELETE` rule. + Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule. For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) @@ -474,7 +474,7 @@ With `WHERE` clause you may specify which of the expired rows to delete or aggre `GROUP BY` expression must be a prefix of the table primary key. -If a column is not part of the `GROUP BY` expression and is not set explicitely in the `SET` clause, in result row it contains an occasional value from the grouped rows (as if aggregate function `any` is applied to it). +If a column is not part of the `GROUP BY` expression and is not set explicitly in the `SET` clause, in result row it contains an occasional value from the grouped rows (as if aggregate function `any` is applied to it). **Examples** diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index 9bfd1816d32..36840e6fd0b 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -96,7 +96,7 @@ SELECT key, sum(value) FROM summtt GROUP BY key When data are inserted into a table, they are saved as-is. ClickHouse merges the inserted parts of data periodically and this is when rows with the same primary key are summed and replaced with one for each resulting part of data. -ClickHouse can merge the data parts so that different resulting parts of data cat consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) an aggregate function [sum()](../../../sql-reference/aggregate-functions/reference/sum.md#agg_function-sum) and `GROUP BY` clause should be used in a query as described in the example above. +ClickHouse can merge the data parts so that different resulting parts of data can consist rows with the same primary key, i.e. the summation will be incomplete. Therefore (`SELECT`) an aggregate function [sum()](../../../sql-reference/aggregate-functions/reference/sum.md#agg_function-sum) and `GROUP BY` clause should be used in a query as described in the example above. ### Common Rules for Summation {#common-rules-for-summation} From 39475e72ddfa67b24f89351e48dffb9d0f8f5ef6 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 14 Jun 2021 18:50:27 +0300 Subject: [PATCH 140/491] updated quantiletdigestweighted.md --- .../aggregate-functions/reference/quantiletdigestweighted.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index 6dce79d8a89..6226f15c9a0 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -21,7 +21,7 @@ toc_priority: 208 quantileTDigestWeighted(level)(expr, weight) ``` -Алиас: `medianTDigest`. +Алиас: `medianTDigestWeighted`. **Аргументы** From 77e048977bb13f9fc05288655420221d7408f1b7 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 14 Jun 2021 19:02:04 +0300 Subject: [PATCH 141/491] updated settings.md --- docs/en/operations/settings/settings.md | 13 +------------ docs/ru/operations/settings/settings.md | 14 +------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index eee27dacceb..2ebb9e0e0a5 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2069,7 +2069,7 @@ Possible values: - Any positive integer. -Default value: 16. +Default value: 128. ## background_fetches_pool_size {#background_fetches_pool_size} @@ -2549,17 +2549,6 @@ Result └──────────────────────────┴───────┴───────────────────────────────────────────────────────┘ ``` -## allow_experimental_bigint_types {#allow_experimental_bigint_types} - -Enables or disables integer values exceeding the range that is supported by the int data type. - -Possible values: - -- 1 — The bigint data type is enabled. -- 0 — The bigint data type is disabled. - -Default value: `0`. - ## persistent {#persistent} Disables persistency for the [Set](../../engines/table-engines/special/set.md#set) and [Join](../../engines/table-engines/special/join.md#join) table engines. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index bbafe428838..848e39d17a8 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2078,7 +2078,7 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; - Положительное целое число. -Значение по умолчанию: 16. +Значение по умолчанию: 128. ## background_fetches_pool_size {#background_fetches_pool_size} @@ -2376,18 +2376,6 @@ SELECT * FROM system.events WHERE event='QueryMemoryLimitExceeded'; └──────────────────────────┴───────┴───────────────────────────────────────────────────────┘ ``` -## allow_experimental_bigint_types {#allow_experimental_bigint_types} - -Включает или отключает поддержку целочисленных значений, превышающих максимальное значение, допустимое для типа `int`. - -Возможные значения: - -- 1 — большие целочисленные значения поддерживаются. -- 0 — большие целочисленные значения не поддерживаются. - -Значение по умолчанию: `0`. - - ## lock_acquire_timeout {#lock_acquire_timeout} Устанавливает, сколько секунд сервер ожидает возможности выполнить блокировку таблицы. From a590a9035bd27cfd5be18c484afdb40fdb5e1f69 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 14 Jun 2021 20:00:24 +0300 Subject: [PATCH 142/491] Updated client-libraries.md --- docs/ru/interfaces/third-party/client-libraries.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 411475f0aaa..b2896f810b7 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -38,6 +38,8 @@ toc_title: "Клиентские библиотеки от сторонних р - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) +- Rust + - [Klickhouse](https://github.com/Protryon/klickhouse) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) From 8a94e9610ea1bd078d90db879eee6edbac7ca6cc Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Mon, 14 Jun 2021 20:19:28 +0300 Subject: [PATCH 143/491] Russian translation --- docs/en/sql-reference/operators/index.md | 23 +---------------------- docs/ru/sql-reference/operators/index.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 8afb77cff8b..966a75d9e98 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -191,9 +191,8 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV You can set the period without using `INTERVAL` by multiplying seconds, minutes, and hours. For example, a period of one day can be set at `60*60*24`. !!! note "Note" - Syntax `now() + 60*60*24` doesn't consider time settings. For example, daylight saving time. + Syntax `now() + 60*60*24` doesn't consider time settings. For example, daylight saving time. The `INTERVAL` syntax or `addDays`-function is always preferred. -The `INTERVAL` syntax or `addDays`-function is always preferred. Examples: @@ -207,26 +206,6 @@ SELECT now() AS current_date_time, current_date_time + 60*60*96; └─────────────────────┴─────────────────────────────────────────────┘ ``` -``` sql -SELECT now() AS current_date_time, current_date_time + 60*60*24 + 60*60*2; -``` - -``` text -┌───current_date_time─┬─plus(plus(now(), multiply(multiply(60, 60), 24)), multiply(multiply(60, 60), 2))─┐ -│ 2021-06-12 17:36:30 │ 2021-06-13 19:36:30 │ -└─────────────────────┴──────────────────────────────────────────────────────────────────────────────────┘ -``` - -``` sql -SELECT now() AS current_date_time, current_date_time + 60*23; -``` - -``` text -┌───current_date_time─┬─plus(now(), multiply(60, 23))─┐ -│ 2021-06-12 17:38:02 │ 2021-06-12 18:01:02 │ -└─────────────────────┴───────────────────────────────┘ -``` - **See Also** - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index b7cacaf7a03..de1c5630255 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -189,6 +189,23 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV └─────────────────────┴────────────────────────────────────────────────────────────┘ ``` +Вы можете задать период без использования синтаксиса `INTERVAL`, умножив секунды, минуты и часы. Например, период в один день можно записать как `60*60*24`. + +!!! note "Примечание" + Синтаксис `now() + 60*60*24` не учитывает региональные настройки времени. Например, переход на летнее время. Для работы с интервалами синтаксис `INTERVAL` или функция `addDays` предпочтительней. + +Пример: + +``` sql +SELECT now() AS current_date_time, current_date_time + 60*60*96; +``` + +``` text +┌───current_date_time─┬─plus(now(), multiply(multiply(60, 60), 96))─┐ +│ 2021-06-12 17:34:49 │ 2021-06-16 17:34:49 │ +└─────────────────────┴─────────────────────────────────────────────┘ +``` + **Смотрите также** - Тип данных [Interval](../../sql-reference/operators/index.md) From 67b7b6fec9e0382c77fe2270e260076620814159 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Mon, 14 Jun 2021 20:36:28 +0300 Subject: [PATCH 144/491] Small fix --- .../aggregate-functions/reference/quantiletdigestweighted.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index 6226f15c9a0..557dafdf49b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -21,7 +21,7 @@ toc_priority: 208 quantileTDigestWeighted(level)(expr, weight) ``` -Алиас: `medianTDigestWeighted`. +Синоним: `medianTDigestWeighted`. **Аргументы** From e96f23d46b37ad32d49a88b8dd9ea87149372e15 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 1 Jun 2021 16:42:56 -0700 Subject: [PATCH 145/491] H3 - add function h3ToGeo --- src/Functions/h3toGeo.cpp | 102 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 104 insertions(+) create mode 100644 src/Functions/h3toGeo.cpp diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp new file mode 100644 index 00000000000..a4e7e2638cb --- /dev/null +++ b/src/Functions/h3toGeo.cpp @@ -0,0 +1,102 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + +} + +namespace +{ + +/// Implements the function h3ToGeo which takes a single argument (h3Index) +/// and returns the latitude and longitude that correspond to the provided h3 index +class FunctionH3ToGeo : public IFunction +{ +public: + static constexpr auto name = "H3ToGeo"; + + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + return std::make_shared( + DataTypes{std::make_shared(), std::make_shared()}, + Strings{"latitude", "longitude"}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_index = arguments[0].column.get(); + + auto dst = ColumnVector::create(); + + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + ColumnPtr res_column; + + + auto latitude = ColumnFloat64::create(input_rows_count); + auto longitude = ColumnFloat64::create(input_rows_count); + + ColumnFloat64::Container & lon_data = longitude->getData(); + ColumnFloat64::Container & lat_data = latitude->getData(); + + for (const auto row : ext::range(0, input_rows_count)) + { + const UInt64 h3Index = col_index->getUInt(row); + GeoCoord coord; + h3ToGeo(h3Index,&coord); + lon_data[row] = coord.lon; + lat_data[row] = coord.lat; + + } + MutableColumns result; + result.emplace_back(std::move(latitude)); + result.emplace_back(std::move(longitude)); + return ColumnTuple::create(std::move(result)); + } +}; + +} + +void registerFunctionH3ToGeo(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 605dd4dcba0..1ab475345a0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -28,6 +28,7 @@ void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); +void registerFunctionH3ToGeo(FunctionFactory &); void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); @@ -66,6 +67,7 @@ void registerFunctionsGeo(FunctionFactory & factory) #if USE_H3 registerFunctionGeoToH3(factory); + registerFunctionH3ToGeo(factory); registerFunctionH3EdgeAngle(factory); registerFunctionH3EdgeLengthM(factory); registerFunctionH3GetResolution(factory); From 99ae86305509d70948cc3353140c63c7ea172868 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 00:00:14 -0700 Subject: [PATCH 146/491] h3ToGeo - few fixes --- src/Functions/h3toGeo.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index a4e7e2638cb..e886248c644 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -32,11 +32,11 @@ namespace { /// Implements the function h3ToGeo which takes a single argument (h3Index) -/// and returns the latitude and longitude that correspond to the provided h3 index +/// and returns the longitude and latitude that correspond to the provided h3 index class FunctionH3ToGeo : public IFunction { public: - static constexpr auto name = "H3ToGeo"; + static constexpr auto name = "h3ToGeo"; static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } @@ -54,7 +54,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared( DataTypes{std::make_shared(), std::make_shared()}, - Strings{"latitude", "longitude"}); + Strings{"longitude", "latitude"}); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override @@ -79,13 +79,13 @@ public: const UInt64 h3Index = col_index->getUInt(row); GeoCoord coord; h3ToGeo(h3Index,&coord); - lon_data[row] = coord.lon; - lat_data[row] = coord.lat; + lon_data[row] = radsToDegs(coord.lon); + lat_data[row] = radsToDegs(coord.lat); } MutableColumns result; - result.emplace_back(std::move(latitude)); result.emplace_back(std::move(longitude)); + result.emplace_back(std::move(latitude)); return ColumnTuple::create(std::move(result)); } }; From 4487745dde2278fdf3bb9d250b926aba0656e119 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 00:07:49 -0700 Subject: [PATCH 147/491] h3ToGeo - docs --- docs/en/sql-reference/functions/geo/h3.md | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 20dc7b29902..6c03f55cebe 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -195,6 +195,41 @@ Result: └────────────────────┘ ``` +## h3ToGeo {#h3togeo} + +Returns `(lon, lat)` that corresponds to the provided H3 index. + +**Syntax** + +``` sql +h3ToGeo(h3Index) +``` + +**Arguments** + +- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned values** + +- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). +- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). + + +**Example** + +Query: + +``` sql +SELECT h3ToGeo(644325524701193974) coordinates; +``` + +Result: + +``` text +┌─coordinates───────────────────────────┐ +│ (37.79506616830252,55.71290243145668) │ +└───────────────────────────────────────┘ +``` ## h3kRing {#h3kring} Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order. From 1284a04a8b957c97e3158a65e2e9a724c3da9e8a Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 09:46:02 -0700 Subject: [PATCH 148/491] h3ToGeo - add tests --- tests/queries/0_stateless/01889_h3_to_geo.reference | 4 ++++ tests/queries/0_stateless/01889_h3_to_geo.sql | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/01889_h3_to_geo.reference create mode 100644 tests/queries/0_stateless/01889_h3_to_geo.sql diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01889_h3_to_geo.reference new file mode 100644 index 00000000000..2821b897dd4 --- /dev/null +++ b/tests/queries/0_stateless/01889_h3_to_geo.reference @@ -0,0 +1,4 @@ +(37.79506616830252,55.71290243145668) +(37.79507532320379,55.712909934159995) +(37.79510864783612,55.71292405516229) +(37.79511780275313,55.712931557864124) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql new file mode 100644 index 00000000000..fc69cec0fb4 --- /dev/null +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +INSERT INTO h3_indexes VALUES(644325524701193974); +INSERT INTO h3_indexes VALUES(644325524701193975); +INSERT INTO h3_indexes VALUES(644325524701193976); +INSERT INTO h3_indexes VALUES(644325524701193977); + +select h3ToGeo(h3_index) from h3_indexes; + +DROP TABLE h3_indexes; \ No newline at end of file From cabd4144da938cbbee70b411c7b42e8c3a1536dc Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 13:20:08 -0700 Subject: [PATCH 149/491] h3ToGeo - fixes to code --- src/Functions/h3toGeo.cpp | 14 +++----------- tests/queries/0_stateless/01889_h3_to_geo.sql | 4 ++-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index e886248c644..be483f8065b 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -24,8 +24,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; - } namespace @@ -61,13 +59,7 @@ public: { const auto * col_index = arguments[0].column.get(); - auto dst = ColumnVector::create(); - - auto & dst_data = dst->getData(); - dst_data.resize(input_rows_count); ColumnPtr res_column; - - auto latitude = ColumnFloat64::create(input_rows_count); auto longitude = ColumnFloat64::create(input_rows_count); @@ -76,13 +68,13 @@ public: for (const auto row : ext::range(0, input_rows_count)) { - const UInt64 h3Index = col_index->getUInt(row); + const UInt64 h3index = col_index->getUInt(row); GeoCoord coord; - h3ToGeo(h3Index,&coord); + h3ToGeo(h3index,&coord); lon_data[row] = radsToDegs(coord.lon); lat_data[row] = radsToDegs(coord.lat); - } + MutableColumns result; result.emplace_back(std::move(longitude)); result.emplace_back(std::move(latitude)); diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql index fc69cec0fb4..04e83a84331 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -7,6 +7,6 @@ INSERT INTO h3_indexes VALUES(644325524701193975); INSERT INTO h3_indexes VALUES(644325524701193976); INSERT INTO h3_indexes VALUES(644325524701193977); -select h3ToGeo(h3_index) from h3_indexes; +select h3ToGeo(h3_index) from h3_indexes order by h3_index; -DROP TABLE h3_indexes; \ No newline at end of file +DROP TABLE h3_indexes; From b17c335074167c263852396a4cea783ccc77cb45 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 6 Jun 2021 15:15:31 -0700 Subject: [PATCH 150/491] h3 - update submodule --- contrib/h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/h3 b/contrib/h3 index e209086ae1b..5c44b06c406 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit e209086ae1b5477307f545a0f6111780edc59940 +Subproject commit 5c44b06c406613b7792a60b11d04b871116f6e30 From 03060518b77533a49aeef37c7dd716cbca2e5c41 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 6 Jun 2021 17:06:59 -0700 Subject: [PATCH 151/491] h3ToGeo - better tests --- .../0_stateless/01659_h3_buffer_overflow.sql | 1 + .../0_stateless/01889_h3_to_geo.reference | 20 ++++++++++++---- tests/queries/0_stateless/01889_h3_to_geo.sql | 24 +++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql index b752059da48..f2d77641ec9 100644 --- a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql +++ b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql @@ -7,3 +7,4 @@ SELECT h3kRing(0xFFFFFFFFFFFFFF, 1000) FORMAT Null; SELECT h3GetBaseCell(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3GetResolution(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3kRing(0xFFFFFFFFFFFFFF, 10) FORMAT Null; +SELECT h3ToGeo(0xFFFFFFFFFFFFFF) FORMAT Null; diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01889_h3_to_geo.reference index 2821b897dd4..5a45c5f3c30 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.reference +++ b/tests/queries/0_stateless/01889_h3_to_geo.reference @@ -1,4 +1,16 @@ -(37.79506616830252,55.71290243145668) -(37.79507532320379,55.712909934159995) -(37.79510864783612,55.71292405516229) -(37.79511780275313,55.712931557864124) +(-173.6412167681162,-14.130272474941535) +(59.48137613600854,58.020407687755686) +(172.68095885060296,-83.6576608516349) +(-94.46556851304558,-69.1999982492279) +(-8.188263637093279,-55.856179102736284) +(77.25594891852249,47.39278564360122) +(135.11348004704536,36.60778126579667) +(39.28534828967223,49.07710003066973) +(124.71163478198051,-27.481172161567258) +(-147.4887686066785,76.73237945824442) +(86.63291906118863,-25.52526285188784) +(23.27751790712118,13.126101362212724) +(-70.40163237204142,-63.12562536833242) +(15.642428355535966,40.285813505163574) +(-76.53411447979884,54.5560449693637) +(8.19906334981474,67.69370966550179) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql index 04e83a84331..f16e03b2bd9 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -2,10 +2,26 @@ DROP TABLE IF EXISTS h3_indexes; CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; -INSERT INTO h3_indexes VALUES(644325524701193974); -INSERT INTO h3_indexes VALUES(644325524701193975); -INSERT INTO h3_indexes VALUES(644325524701193976); -INSERT INTO h3_indexes VALUES(644325524701193977); +-- values are random h3 indexes at various resolutions from 0 to 15. +-- generated using h3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c + +INSERT INTO h3_indexes VALUES (579205133326352383); +INSERT INTO h3_indexes VALUES (581263419093549055); +INSERT INTO h3_indexes VALUES (589753847883235327); +INSERT INTO h3_indexes VALUES (594082350283882495); +INSERT INTO h3_indexes VALUES (598372386957426687); +INSERT INTO h3_indexes VALUES (599542359671177215); +INSERT INTO h3_indexes VALUES (604296355086598143); +INSERT INTO h3_indexes VALUES (608785214872748031); +INSERT INTO h3_indexes VALUES (615732192485572607); +INSERT INTO h3_indexes VALUES (617056794467368959); +INSERT INTO h3_indexes VALUES (624586477873168383); +INSERT INTO h3_indexes VALUES (627882919484481535); +INSERT INTO h3_indexes VALUES (634600058503392255); +INSERT INTO h3_indexes VALUES (635544851677385791); +INSERT INTO h3_indexes VALUES (639763125756281263); +INSERT INTO h3_indexes VALUES (644178757620501158); + select h3ToGeo(h3_index) from h3_indexes order by h3_index; From a910a15115e0feaf9b1ccfe802bc138fb55a47a3 Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Mon, 14 Jun 2021 23:05:37 +0300 Subject: [PATCH 152/491] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 966a75d9e98..c26a13e1ae6 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -191,7 +191,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV You can set the period without using `INTERVAL` by multiplying seconds, minutes, and hours. For example, a period of one day can be set at `60*60*24`. !!! note "Note" - Syntax `now() + 60*60*24` doesn't consider time settings. For example, daylight saving time. The `INTERVAL` syntax or `addDays`-function is always preferred. + The `INTERVAL` syntax or `addDays` function are always preferred. Simple addition or subtraction (syntax like `now() + ...`) doesn't consider time settings. For example, daylight saving time. Examples: @@ -313,4 +313,3 @@ SELECT * FROM t_null WHERE y IS NOT NULL │ 2 │ 3 │ └───┴───┘ ``` - From e7bd849f962ad901b811106f0ed610f156f2e6a8 Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Mon, 14 Jun 2021 23:05:46 +0300 Subject: [PATCH 153/491] Update docs/ru/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/operators/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index de1c5630255..201cca54142 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -192,7 +192,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV Вы можете задать период без использования синтаксиса `INTERVAL`, умножив секунды, минуты и часы. Например, период в один день можно записать как `60*60*24`. !!! note "Примечание" - Синтаксис `now() + 60*60*24` не учитывает региональные настройки времени. Например, переход на летнее время. Для работы с интервалами синтаксис `INTERVAL` или функция `addDays` предпочтительней. + Для работы с датами лучше использовать синтаксис `INTERVAL` или функцию `addDays`. Простое сложение (например, синтаксис `now() + ...`) не учитывает региональные настройки времени, например, переход на летнее время. Пример: @@ -313,4 +313,3 @@ SELECT * FROM t_null WHERE y IS NOT NULL │ 2 │ 3 │ └───┴───┘ ``` - From ac86db5ef4e1ace203647a4af9bca131025ce77b Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Mon, 14 Jun 2021 23:05:53 +0300 Subject: [PATCH 154/491] Update docs/ru/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 201cca54142..ca742996a94 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -189,7 +189,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV └─────────────────────┴────────────────────────────────────────────────────────────┘ ``` -Вы можете задать период без использования синтаксиса `INTERVAL`, умножив секунды, минуты и часы. Например, период в один день можно записать как `60*60*24`. +Вы можете изменить дату, не используя синтаксис `INTERVAL`, а просто добавив или отняв секунды, минуты и часы. Например, чтобы передвинуть дату на один день вперед, можно прибавить к ней значение `60*60*24`. !!! note "Примечание" Для работы с датами лучше использовать синтаксис `INTERVAL` или функцию `addDays`. Простое сложение (например, синтаксис `now() + ...`) не учитывает региональные настройки времени, например, переход на летнее время. From 8531aef8bd316e4e9dea4b9fbe08c4bb561bd51c Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Mon, 14 Jun 2021 23:06:04 +0300 Subject: [PATCH 155/491] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index c26a13e1ae6..58ab0068c52 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -188,7 +188,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV └─────────────────────┴────────────────────────────────────────────────────────────┘ ``` -You can set the period without using `INTERVAL` by multiplying seconds, minutes, and hours. For example, a period of one day can be set at `60*60*24`. +You can work with dates without using `INTERVAL`, just by adding or subtracting seconds, minutes, and hours. For example, an interval of one day can be set by adding `60*60*24`. !!! note "Note" The `INTERVAL` syntax or `addDays` function are always preferred. Simple addition or subtraction (syntax like `now() + ...`) doesn't consider time settings. For example, daylight saving time. From daf020347e20029a754793dddf1db6ad4b4c47e0 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Mon, 14 Jun 2021 18:10:46 -0300 Subject: [PATCH 156/491] test for `PARTITION BY 0 * id` --- ...06_partition_by_multiply_by_zero.reference | 2 ++ .../01906_partition_by_multiply_by_zero.sql | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/01906_partition_by_multiply_by_zero.reference create mode 100644 tests/queries/0_stateless/01906_partition_by_multiply_by_zero.sql diff --git a/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.reference b/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.reference new file mode 100644 index 00000000000..0d62b2692ed --- /dev/null +++ b/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.reference @@ -0,0 +1,2 @@ +58 +58 diff --git a/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.sql b/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.sql new file mode 100644 index 00000000000..be890339c5f --- /dev/null +++ b/tests/queries/0_stateless/01906_partition_by_multiply_by_zero.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS t_01906; + +CREATE TABLE t_01906 +( + `id` UInt64, + `update_ts` DateTime, + `value` UInt32 +) +ENGINE = ReplacingMergeTree(update_ts) +PARTITION BY 0 * id +ORDER BY (update_ts, id); + +INSERT INTO t_01906 SELECT + number, + toDateTime('2020-01-01 00:00:00'), + 1 +FROM numbers(100); + +SELECT count() FROM t_01906 WHERE id >= 42; + +SELECT count() FROM t_01906 FINAL WHERE id >= 42 and update_ts <= '2021-01-01 00:00:00'; + +DROP TABLE t_01906; From 22bd65996ce992ba81aa92f552f6b75480e6ab6a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 00:58:29 +0300 Subject: [PATCH 157/491] Fix TOCTOU in Install --- programs/install/Install.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index a7f566a78b8..8a32f542ceb 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -819,15 +819,25 @@ namespace if (fs::exists(pid_file)) { - ReadBufferFromFile in(pid_file.string()); - if (tryReadIntText(pid, in)) + try { - fmt::print("{} file exists and contains pid = {}.\n", pid_file.string(), pid); + ReadBufferFromFile in(pid_file.string()); + if (tryReadIntText(pid, in)) + { + fmt::print("{} file exists and contains pid = {}.\n", pid_file.string(), pid); + } + else + { + fmt::print("{} file exists but damaged, ignoring.\n", pid_file.string()); + fs::remove(pid_file); + } } - else + catch (const Exception & e) { - fmt::print("{} file exists but damaged, ignoring.\n", pid_file.string()); - fs::remove(pid_file); + if (e.code() != ErrorCodes::FILE_DOESNT_EXIST) + throw; + + /// If file does not exist (TOCTOU) - it's ok. } } From 62d4c51524d2b1574f49d5155b2a7c2f1f5a4dd1 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 14 Jun 2021 18:19:36 -0400 Subject: [PATCH 158/491] Updating tests to support new preprocessed configs name that start with an underscore. --- tests/testflows/ldap/authentication/tests/common.py | 4 ++-- tests/testflows/ldap/external_user_directory/tests/common.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index ec6a66c0257..0f36879ef62 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -92,7 +92,7 @@ def add_config(config, timeout=300, restart=False, modify=False): """Check that preprocessed config is updated. """ started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + command = f"cat /var/lib/clickhouse/preprocessed_configs/_{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode @@ -105,7 +105,7 @@ def add_config(config, timeout=300, restart=False, modify=False): time.sleep(1) if settings.debug: - node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}") + node.command(f"cat /var/lib/clickhouse/preprocessed_configs/_{config.preprocessed_name}") if after_removal: assert exitcode == 1, error() diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index f6d1654efd6..f23356bd061 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -138,7 +138,7 @@ def invalid_ldap_external_user_directory_config(server, roles, message, tail=30, with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + command = f"cat /var/lib/clickhouse/preprocessed_configs/_{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: From 07f0ee39ab84b90b043a2835aed78485cc0498d6 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 14 Jun 2021 18:20:19 -0400 Subject: [PATCH 159/491] Enabling LDAP tests. --- tests/testflows/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index c868d4a0d92..7eb170a0002 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -23,7 +23,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): with Pool(8) as pool: try: run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) #run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) From 33f270c2dceb4eab3522a61ef76de1f5d6126edf Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 14 Jun 2021 19:00:29 -0400 Subject: [PATCH 160/491] Adding test to check adding LDAP user to an LDAP group that maps to an unknown RBAC role while already having other role being mapped. --- .../ldap/role_mapping/tests/mapping.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/testflows/ldap/role_mapping/tests/mapping.py b/tests/testflows/ldap/role_mapping/tests/mapping.py index 8076865ab1f..ccdde9c06c8 100644 --- a/tests/testflows/ldap/role_mapping/tests/mapping.py +++ b/tests/testflows/ldap/role_mapping/tests/mapping.py @@ -624,6 +624,82 @@ def role_not_present(self, ldap_server, ldap_user): with And("the user not to have any mapped LDAP role"): assert r.output == "", error() +@TestScenario +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent("1.0") +) +def add_new_role_not_present(self, ldap_server, ldap_user): + """Check that LDAP user can still authenticate when the LDAP + user is added to a new LDAP group that does not match any existing + RBAC roles while having other role being already mapped. + """ + uid = getuid() + role_name = f"role_{uid}" + + role_mappings = [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", + "prefix": "clickhouse_" + } + ] + + with Given("I add LDAP group"): + groups = add_ldap_groups(groups=({"cn": "clickhouse_" + role_name},)) + + with And("I add LDAP user to the group"): + add_user_to_group_in_ldap(user=ldap_user, group=groups[0]) + + with And("I add matching RBAC role"): + roles = add_rbac_roles(roles=(f"{role_name}",)) + + with And("I add LDAP external user directory configuration"): + add_ldap_external_user_directory(server=ldap_server, + role_mappings=role_mappings, restart=True) + + with When(f"I login as an LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + + with Then("I expect the login to succeed"): + assert r.exitcode == 0, error() + + with And("the user should have the mapped LDAP role"): + assert f"{role_name}" in r.output, error() + + with When("I add LDAP group that maps to unknown role"): + unknown_groups = add_ldap_groups(groups=({"cn": "clickhouse_" + role_name + "_unknown"},)) + + with And("I add LDAP user to the group that maps to unknown role"): + add_user_to_group_in_ldap(user=ldap_user, group=unknown_groups[0]) + + with And(f"I again login as an LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + + with Then("I expect the login to succeed"): + assert r.exitcode == 0, error() + + with And("the user should still have the present mapped LDAP role"): + assert f"{role_name}" in r.output, error() + + with When("I add matching previously unknown RBAC role"): + unknown_roles = add_rbac_roles(roles=(f"{role_name}_unknown",)) + + with And(f"I again login as an LDAP user after previously unknown RBAC role has been added"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + + with Then("I expect the login to succeed"): + assert r.exitcode == 0, error() + + with And("the user should still have the first mapped LDAP role"): + assert f"{role_name}" in r.output, error() + + with And("the user should have the previously unknown mapped LDAP role"): + assert f"{role_name}_unknown" in r.output, error() + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Removed("1.0"), From 78b61237907d99df209ab174cf29fd88ab99b07e Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 13 Jun 2021 09:14:29 -0700 Subject: [PATCH 161/491] h3ToGeo - rename test files --- .../{01889_h3_to_geo.reference => 01906_h3_to_geo.reference} | 0 .../0_stateless/{01889_h3_to_geo.sql => 01906_h3_to_geo.sql} | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/queries/0_stateless/{01889_h3_to_geo.reference => 01906_h3_to_geo.reference} (100%) rename tests/queries/0_stateless/{01889_h3_to_geo.sql => 01906_h3_to_geo.sql} (75%) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01906_h3_to_geo.reference similarity index 100% rename from tests/queries/0_stateless/01889_h3_to_geo.reference rename to tests/queries/0_stateless/01906_h3_to_geo.reference diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01906_h3_to_geo.sql similarity index 75% rename from tests/queries/0_stateless/01889_h3_to_geo.sql rename to tests/queries/0_stateless/01906_h3_to_geo.sql index f16e03b2bd9..c78270d2808 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01906_h3_to_geo.sql @@ -2,8 +2,8 @@ DROP TABLE IF EXISTS h3_indexes; CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; --- values are random h3 indexes at various resolutions from 0 to 15. --- generated using h3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c +-- Random geo coordinates were generated using the H3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c at various resolutions from 0 to 15. +-- Corresponding H3 index values were in turn generated with those geo coordinates using `geoToH3(lon, lat, res)` ClickHouse function for the following test. INSERT INTO h3_indexes VALUES (579205133326352383); INSERT INTO h3_indexes VALUES (581263419093549055); From d073161455b9b11ef7c2b933c975a0a941ae71ba Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 14 Jun 2021 11:36:41 -0700 Subject: [PATCH 162/491] h3ToGeo - try fix msan fuzzer errors --- src/Functions/h3toGeo.cpp | 74 ++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index be483f8065b..8ce0b15dc8c 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -23,7 +23,7 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace @@ -36,50 +36,52 @@ class FunctionH3ToGeo : public IFunction public: static constexpr auto name = "h3ToGeo"; - static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return std::make_shared(); } std::string getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared( + DataTypes{std::make_shared(), std::make_shared()}, + Strings{"longitude", "latitude"}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_index = arguments[0].column.get(); + + auto latitude = ColumnFloat64::create(input_rows_count); + auto longitude = ColumnFloat64::create(input_rows_count); + + ColumnFloat64::Container & lon_data = longitude->getData(); + ColumnFloat64::Container & lat_data = latitude->getData(); + + + for (size_t row = 0; row < input_rows_count; ++row) { - const auto * arg = arguments[0].get(); - if (!WhichDataType(arg).isUInt64()) - throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared( - DataTypes{std::make_shared(), std::make_shared()}, - Strings{"longitude", "latitude"}); + H3Index h3index = col_index->getUInt(row); + GeoCoord coord{}; + + h3ToGeo(h3index,&coord); + lon_data[row] = radsToDegs(coord.lon); + lat_data[row] = radsToDegs(coord.lat); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const auto * col_index = arguments[0].column.get(); - - ColumnPtr res_column; - auto latitude = ColumnFloat64::create(input_rows_count); - auto longitude = ColumnFloat64::create(input_rows_count); - - ColumnFloat64::Container & lon_data = longitude->getData(); - ColumnFloat64::Container & lat_data = latitude->getData(); - - for (const auto row : ext::range(0, input_rows_count)) - { - const UInt64 h3index = col_index->getUInt(row); - GeoCoord coord; - h3ToGeo(h3index,&coord); - lon_data[row] = radsToDegs(coord.lon); - lat_data[row] = radsToDegs(coord.lat); - } - - MutableColumns result; - result.emplace_back(std::move(longitude)); - result.emplace_back(std::move(latitude)); - return ColumnTuple::create(std::move(result)); - } + MutableColumns columns; + columns.emplace_back(std::move(longitude)); + columns.emplace_back(std::move(latitude)); + return ColumnTuple::create(std::move(columns)); + } }; } From c12eebb5668fdcd48c96a117ef8b72f43fce45b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 03:28:57 +0300 Subject: [PATCH 163/491] Add a test --- .../01906_bigint_accurate_cast_ubsan.reference | 5 +++++ .../01906_bigint_accurate_cast_ubsan.sql | 15 +++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.reference create mode 100644 tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.sql diff --git a/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.reference b/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.reference new file mode 100644 index 00000000000..a293d9344f8 --- /dev/null +++ b/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.reference @@ -0,0 +1,5 @@ +10000000000000000000 +10000000000000000000 +10000000000000000000 +10000000000000000000 +10000000000000000000 diff --git a/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.sql b/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.sql new file mode 100644 index 00000000000..4b9fa9662a9 --- /dev/null +++ b/tests/queries/0_stateless/01906_bigint_accurate_cast_ubsan.sql @@ -0,0 +1,15 @@ +SELECT accurateCast(1e35, 'UInt32'); -- { serverError 70 } +SELECT accurateCast(1e35, 'UInt64'); -- { serverError 70 } +SELECT accurateCast(1e35, 'UInt128'); -- { serverError 70 } +SELECT accurateCast(1e35, 'UInt256'); -- { serverError 70 } + +SELECT accurateCast(1e19, 'UInt64'); +SELECT accurateCast(1e19, 'UInt128'); +SELECT accurateCast(1e19, 'UInt256'); +SELECT accurateCast(1e20, 'UInt64'); -- { serverError 70 } +SELECT accurateCast(1e20, 'UInt128'); -- { serverError 70 } +SELECT accurateCast(1e20, 'UInt256'); -- { serverError 70 } + +SELECT accurateCast(1e19, 'Int64'); -- { serverError 70 } +SELECT accurateCast(1e19, 'Int128'); +SELECT accurateCast(1e19, 'Int256'); From c5181cf8970af9870d13c8f465e11e9a5552f00e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 03:29:20 +0300 Subject: [PATCH 164/491] Fix wrong code in wide_int --- base/common/wide_integer.h | 5 +---- base/common/wide_integer_impl.h | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/base/common/wide_integer.h b/base/common/wide_integer.h index 419b4e4558c..de349633723 100644 --- a/base/common/wide_integer.h +++ b/base/common/wide_integer.h @@ -109,10 +109,7 @@ public: constexpr explicit operator bool() const noexcept; - template - using _integral_not_wide_integer_class = typename std::enable_if::value, T>::type; - - template > + template , T>> constexpr operator T() const noexcept; constexpr operator long double() const noexcept; diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index 725caec6a3e..ad1d994492e 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -255,13 +255,13 @@ struct integer::_impl set_multiplier(self, alpha); self *= max_int; - self += static_cast(t - alpha * static_cast(max_int)); // += b_i + self += static_cast(t - static_cast(alpha) * static_cast(max_int)); // += b_i } - constexpr static void wide_integer_from_builtin(integer& self, double rhs) noexcept + constexpr static void wide_integer_from_builtin(integer & self, double rhs) noexcept { constexpr int64_t max_int = std::numeric_limits::max(); - constexpr int64_t min_int = std::numeric_limits::min(); + constexpr int64_t min_int = std::numeric_limits::lowest(); /// There are values in int64 that have more than 53 significant bits (in terms of double /// representation). Such values, being promoted to double, are rounded up or down. If they are rounded up, @@ -271,14 +271,14 @@ struct integer::_impl /// The necessary check here is that long double has enough significant (mantissa) bits to store the /// int64_t max value precisely. - //TODO Be compatible with Apple aarch64 + // TODO Be compatible with Apple aarch64 #if not (defined(__APPLE__) && defined(__aarch64__)) static_assert(LDBL_MANT_DIG >= 64, - "On your system long double has less than 64 precision bits," + "On your system long double has less than 64 precision bits, " "which may result in UB when initializing double from int64_t"); #endif - if ((rhs > 0 && rhs < static_cast(max_int)) || (rhs < 0 && rhs > static_cast(min_int))) + if (rhs > static_cast(min_int) && rhs < static_cast(max_int)) { self = static_cast(rhs); return; From 6eb06d84d41fd97cff6a74d9a4b4517f53aca296 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 03:29:44 +0300 Subject: [PATCH 165/491] Fix decomposed float --- base/common/DecomposedFloat.h | 60 ++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/base/common/DecomposedFloat.h b/base/common/DecomposedFloat.h index 078ba823c15..0b283f1ef6f 100644 --- a/base/common/DecomposedFloat.h +++ b/base/common/DecomposedFloat.h @@ -91,10 +91,12 @@ struct DecomposedFloat /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic. + /// This function is generic, big integers (128, 256 bit) are supported as well. /// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers. /// (note that we need total order) + /// Returns -1, 0 or 1. template - int compare(Int rhs) + int compare(Int rhs) const { if (rhs == 0) return sign(); @@ -137,10 +139,11 @@ struct DecomposedFloat if (normalized_exponent() >= static_cast(8 * sizeof(Int) - is_signed_v)) return is_negative() ? -1 : 1; - using UInt = make_unsigned_t; + using UInt = std::conditional_t<(sizeof(Int) > sizeof(typename Traits::UInt)), make_unsigned_t, typename Traits::UInt>; UInt uint_rhs = rhs < 0 ? -rhs : rhs; /// Smaller octave: abs(rhs) < abs(float) + /// FYI, TIL: octave is also called "binade", https://en.wikipedia.org/wiki/Binade if (uint_rhs < (static_cast(1) << normalized_exponent())) return is_negative() ? -1 : 1; @@ -154,11 +157,11 @@ struct DecomposedFloat bool large_and_always_integer = normalized_exponent() >= static_cast(Traits::mantissa_bits); - typename Traits::UInt a = large_and_always_integer - ? mantissa() << (normalized_exponent() - Traits::mantissa_bits) - : mantissa() >> (Traits::mantissa_bits - normalized_exponent()); + UInt a = large_and_always_integer + ? static_cast(mantissa()) << (normalized_exponent() - Traits::mantissa_bits) + : static_cast(mantissa()) >> (Traits::mantissa_bits - normalized_exponent()); - typename Traits::UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); + UInt b = uint_rhs - (static_cast(1) << normalized_exponent()); if (a < b) return is_negative() ? 1 : -1; @@ -175,40 +178,73 @@ struct DecomposedFloat template - bool equals(Int rhs) + bool equals(Int rhs) const { return compare(rhs) == 0; } template - bool notEquals(Int rhs) + bool notEquals(Int rhs) const { return compare(rhs) != 0; } template - bool less(Int rhs) + bool less(Int rhs) const { return compare(rhs) < 0; } template - bool greater(Int rhs) + bool greater(Int rhs) const { return compare(rhs) > 0; } template - bool lessOrEquals(Int rhs) + bool lessOrEquals(Int rhs) const { return compare(rhs) <= 0; } template - bool greaterOrEquals(Int rhs) + bool greaterOrEquals(Int rhs) const { return compare(rhs) >= 0; } + + + template + Int toInt() const + { + /// sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)) + + /// Too large exponent, implementation specific behaviour. Includes infs and NaNs. + if (normalized_exponent() >= sizeof(Int) * 8) + return is_negative() ? std::numeric_limits::lowest() : std::numeric_limits::max(); + + if (normalized_exponent() < 0) + return 0; + + Int res{1}; + res <<= normalized_exponent(); + + if (normalized_exponent() >= static_cast(Traits::mantissa_bits)) + { + res += static_cast(mantissa()) << (normalized_exponent() - Traits::mantissa_bits); + } + else + { + /// NOTE rounding towards zero, it can be different to current CPU rounding mode. + res += static_cast(mantissa()) >> (Traits::mantissa_bits - normalized_exponent()); + } + + /// Avoid UB on negation of the most negative numbers. Also implementation specific behaviour for unsigned integers. + if (is_negative() && res != std::numeric_limits::lowest()) + res = -res; + + return res; + } }; From 3ee26c822dde660bc074c3a0fc418efd069cb2b4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 03:30:01 +0300 Subject: [PATCH 166/491] Remove unused function --- base/common/DecomposedFloat.h | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/base/common/DecomposedFloat.h b/base/common/DecomposedFloat.h index 0b283f1ef6f..21034908fe7 100644 --- a/base/common/DecomposedFloat.h +++ b/base/common/DecomposedFloat.h @@ -212,39 +212,6 @@ struct DecomposedFloat { return compare(rhs) >= 0; } - - - template - Int toInt() const - { - /// sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)) - - /// Too large exponent, implementation specific behaviour. Includes infs and NaNs. - if (normalized_exponent() >= sizeof(Int) * 8) - return is_negative() ? std::numeric_limits::lowest() : std::numeric_limits::max(); - - if (normalized_exponent() < 0) - return 0; - - Int res{1}; - res <<= normalized_exponent(); - - if (normalized_exponent() >= static_cast(Traits::mantissa_bits)) - { - res += static_cast(mantissa()) << (normalized_exponent() - Traits::mantissa_bits); - } - else - { - /// NOTE rounding towards zero, it can be different to current CPU rounding mode. - res += static_cast(mantissa()) >> (Traits::mantissa_bits - normalized_exponent()); - } - - /// Avoid UB on negation of the most negative numbers. Also implementation specific behaviour for unsigned integers. - if (is_negative() && res != std::numeric_limits::lowest()) - res = -res; - - return res; - } }; From b152d7589fced33b37e24e68ee192c486fd98786 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 14 Jun 2021 10:35:12 +0000 Subject: [PATCH 167/491] Bridge contsraints --- base/bridge/IBridge.cpp | 30 +++++++++++++++++++ programs/install/Install.cpp | 58 ++++++++++++++++++++++++++---------- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/base/bridge/IBridge.cpp b/base/bridge/IBridge.cpp index b2ec53158b1..c3f63d871d3 100644 --- a/base/bridge/IBridge.cpp +++ b/base/bridge/IBridge.cpp @@ -8,7 +8,12 @@ #include #include #include +#include #include +#include +#include +#include +#include #if USE_ODBC # include @@ -163,6 +168,31 @@ void IBridge::initialize(Application & self) max_server_connections = config().getUInt("max-server-connections", 1024); keep_alive_timeout = config().getUInt64("keep-alive-timeout", 10); + struct rlimit limit; + const UInt64 gb = 1024 * 1024 * 1024; + + /// Set maximum RSS to 1 GiB. + limit.rlim_max = limit.rlim_cur = gb; + if (setrlimit(RLIMIT_RSS, &limit)) + LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})", + errnoToString(errno), limit.rlim_cur, limit.rlim_max); + + if (!getrlimit(RLIMIT_RSS, &limit)) + LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max); + + try + { + const auto oom_score = toString(config().getUInt64("bridge_oom_score", 500)); + WriteBufferFromFile buf("/proc/self/oom_score_adj"); + buf.write(oom_score.data(), oom_score.size()); + buf.close(); + LOG_INFO(log, "OOM score is set to {}", oom_score); + } + catch (const Exception & e) + { + LOG_WARNING(log, "Failed to set OOM score, error: {}", e.what()); + } + initializeTerminationAndSignalProcessing(); ServerApplication::initialize(self); // NOLINT diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index a7f566a78b8..0bd54bea5de 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -75,6 +75,9 @@ namespace ErrorCodes #define HILITE "\033[1m" #define END_HILITE "\033[0m" +static constexpr auto CLICKHOUSE_BRIDGE_USER = "clickhouse-bridge"; +static constexpr auto CLICKHOUSE_BRIDGE_GROUP = "clickhouse-bridge"; + using namespace DB; namespace po = boost::program_options; namespace fs = std::filesystem; @@ -150,7 +153,6 @@ int mainEntryClickHouseInstall(int argc, char ** argv) << argv[0] << " install [options]\n"; std::cout << desc << '\n'; - return 1; } try @@ -324,26 +326,34 @@ int mainEntryClickHouseInstall(int argc, char ** argv) std::string user = options["user"].as(); std::string group = options["group"].as(); + auto create_group = [](const String & group_name) + { + std::string command = fmt::format("groupadd -r {}", group_name); + fmt::print(" {}\n", command); + executeScript(command); + }; + if (!group.empty()) { - { - fmt::print("Creating clickhouse group if it does not exist.\n"); - std::string command = fmt::format("groupadd -r {}", group); - fmt::print(" {}\n", command); - executeScript(command); - } + fmt::print("Creating clickhouse group if it does not exist.\n"); + create_group(group); } else fmt::print("Will not create clickhouse group"); + auto create_user = [](const String & user_name, const String & group_name) + { + std::string command = group_name.empty() + ? fmt::format("useradd -r --shell /bin/false --home-dir /nonexistent --user-group {}", user_name) + : fmt::format("useradd -r --shell /bin/false --home-dir /nonexistent -g {} {}", group_name, user_name); + fmt::print(" {}\n", command); + executeScript(command); + }; + if (!user.empty()) { fmt::print("Creating clickhouse user if it does not exist.\n"); - std::string command = group.empty() - ? fmt::format("useradd -r --shell /bin/false --home-dir /nonexistent --user-group {}", user) - : fmt::format("useradd -r --shell /bin/false --home-dir /nonexistent -g {} {}", group, user); - fmt::print(" {}\n", command); - executeScript(command); + create_user(user, group); if (group.empty()) group = user; @@ -475,12 +485,15 @@ int mainEntryClickHouseInstall(int argc, char ** argv) } } - /// Chmod and chown configs + auto change_ownership = [](const String & file_name, const String & user_name, const String & group_name) { - std::string command = fmt::format("chown --recursive {}:{} '{}'", user, group, config_dir.string()); + std::string command = fmt::format("chown --recursive {}:{} '{}'", user_name, group_name, file_name); fmt::print(" {}\n", command); executeScript(command); - } + }; + + /// Chmod and chown configs + change_ownership(config_dir.string(), user, group); /// Symlink "preprocessed_configs" is created by the server, so "write" is needed. fs::permissions(config_dir, fs::perms::owner_all, fs::perm_options::replace); @@ -558,7 +571,19 @@ int mainEntryClickHouseInstall(int argc, char ** argv) /// Data directory is not accessible to anyone except clickhouse. fs::permissions(data_path, fs::perms::owner_all, fs::perm_options::replace); - /// Set up password for default user. + fs::path odbc_bridge_path = bin_dir / "clickhouse-odbc-bridge"; + fs::path library_bridge_path = bin_dir / "clickhouse-library-bridge"; + + if (fs::exists(odbc_bridge_path) || fs::exists(library_bridge_path)) + { + create_group(CLICKHOUSE_BRIDGE_GROUP); + create_user(CLICKHOUSE_BRIDGE_USER, CLICKHOUSE_BRIDGE_GROUP); + + if (fs::exists(odbc_bridge_path)) + change_ownership(odbc_bridge_path, CLICKHOUSE_BRIDGE_USER, CLICKHOUSE_BRIDGE_GROUP); + if (fs::exists(library_bridge_path)) + change_ownership(library_bridge_path, CLICKHOUSE_BRIDGE_USER, CLICKHOUSE_BRIDGE_GROUP); + } bool stdin_is_a_tty = isatty(STDIN_FILENO); bool stdout_is_a_tty = isatty(STDOUT_FILENO); @@ -573,6 +598,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) /// We can ask password even if stdin is closed/redirected but /dev/tty is available. bool can_ask_password = !noninteractive && stdout_is_a_tty; + /// Set up password for default user. if (has_password_for_default_user) { fmt::print(HILITE "Password for default user is already specified. To remind or reset, see {} and {}." END_HILITE "\n", From e3653e70da7cdef26ae7b73b5f7544502890be97 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 04:43:35 +0300 Subject: [PATCH 168/491] Update prompt in client when reconnecting --- programs/client/Client.cpp | 126 +++++++++++++++++++------------------ src/Client/Connection.h | 3 + 2 files changed, 68 insertions(+), 61 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 49d3dc3d10b..1d0d789a29a 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -549,65 +549,6 @@ private: /// Initialize DateLUT here to avoid counting time spent here as query execution time. const auto local_tz = DateLUT::instance().getTimeZone(); - if (!context->getSettingsRef().use_client_time_zone) - { - const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts); - if (!time_zone.empty()) - { - try - { - DateLUT::setDefaultTimezone(time_zone); - } - catch (...) - { - std::cerr << "Warning: could not switch to server time zone: " << time_zone - << ", reason: " << getCurrentExceptionMessage(/* with_stacktrace = */ false) << std::endl - << "Proceeding with local time zone." << std::endl - << std::endl; - } - } - else - { - std::cerr << "Warning: could not determine server time zone. " - << "Proceeding with local time zone." << std::endl - << std::endl; - } - } - - Strings keys; - - prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) "); - - config().keys("prompt_by_server_display_name", keys); - - for (const String & key : keys) - { - if (key != "default" && server_display_name.find(key) != std::string::npos) - { - prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name." + key); - break; - } - } - - /// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color. - { - String unescaped_prompt_by_server_display_name; - ReadBufferFromString in(prompt_by_server_display_name); - readEscapedString(unescaped_prompt_by_server_display_name, in); - prompt_by_server_display_name = std::move(unescaped_prompt_by_server_display_name); - } - - /// Prompt may contain the following substitutions in a form of {name}. - std::map prompt_substitutions{ - {"host", connection_parameters.host}, - {"port", toString(connection_parameters.port)}, - {"user", connection_parameters.user}, - {"display_name", server_display_name}, - }; - - /// Quite suboptimal. - for (const auto & [key, value] : prompt_substitutions) - boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value); if (is_interactive) { @@ -805,6 +746,66 @@ private: << std::endl; } } + + if (!context->getSettingsRef().use_client_time_zone) + { + const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts); + if (!time_zone.empty()) + { + try + { + DateLUT::setDefaultTimezone(time_zone); + } + catch (...) + { + std::cerr << "Warning: could not switch to server time zone: " << time_zone + << ", reason: " << getCurrentExceptionMessage(/* with_stacktrace = */ false) << std::endl + << "Proceeding with local time zone." << std::endl + << std::endl; + } + } + else + { + std::cerr << "Warning: could not determine server time zone. " + << "Proceeding with local time zone." << std::endl + << std::endl; + } + } + + Strings keys; + + prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) "); + + config().keys("prompt_by_server_display_name", keys); + + for (const String & key : keys) + { + if (key != "default" && server_display_name.find(key) != std::string::npos) + { + prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name." + key); + break; + } + } + + /// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color. + { + String unescaped_prompt_by_server_display_name; + ReadBufferFromString in(prompt_by_server_display_name); + readEscapedString(unescaped_prompt_by_server_display_name, in); + prompt_by_server_display_name = std::move(unescaped_prompt_by_server_display_name); + } + + /// Prompt may contain the following substitutions in a form of {name}. + std::map prompt_substitutions{ + {"host", connection_parameters.host}, + {"port", toString(connection_parameters.port)}, + {"user", connection_parameters.user}, + {"display_name", server_display_name}, + }; + + /// Quite suboptimal. + for (const auto & [key, value] : prompt_substitutions) + boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value); } @@ -1202,7 +1203,9 @@ private: client_exception.reset(); server_exception.reset(); have_error = false; - connection->forceConnected(connection_parameters.timeouts); + + if (!connection->checkConnected()) + connect(); } // Report error. @@ -1603,7 +1606,8 @@ private: if (with_output && with_output->settings_ast) apply_query_settings(*with_output->settings_ast); - connection->forceConnected(connection_parameters.timeouts); + if (!connection->checkConnected()) + connect(); ASTPtr input_function; if (insert && insert->select) diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 80dbd9ed44e..2ea5a236a13 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -185,6 +185,9 @@ public: bool isConnected() const { return connected; } + /// Check if connection is still active with ping request. + bool checkConnected() { return connected && ping(); } + TablesStatusResponse getTablesStatus(const ConnectionTimeouts & timeouts, const TablesStatusRequest & request); From c41b58b148ab56727a62d8ce607639ca2fbe3466 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 06:52:49 +0300 Subject: [PATCH 169/491] Fix UBSan report --- base/common/wide_integer_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index ad1d994492e..d2ef8b22d65 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -255,7 +255,7 @@ struct integer::_impl set_multiplier(self, alpha); self *= max_int; - self += static_cast(t - static_cast(alpha) * static_cast(max_int)); // += b_i + self += static_cast(t - floor(alpha) * static_cast(max_int)); // += b_i } constexpr static void wide_integer_from_builtin(integer & self, double rhs) noexcept From 0f9eb979aa859f323ec9c57214e7acfde9f216ef Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 06:54:24 +0300 Subject: [PATCH 170/491] Remove code and fix #20549 --- src/Functions/if.cpp | 131 ++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 69 deletions(-) diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index ec3447ffb81..8b930a73dfb 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -153,21 +153,6 @@ struct NumIfImpl, Decimal, Decimal> } }; -template -struct NumIfImpl -{ -private: - [[noreturn]] static void throwError() - { - throw Exception("Incompatible types of arguments corresponding to two conditional branches", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } -public: - template static ColumnPtr vectorVector(Args &&...) { throwError(); } - template static ColumnPtr vectorConstant(Args &&...) { throwError(); } - template static ColumnPtr constantVector(Args &&...) { throwError(); } - template static ColumnPtr constantConstant(Args &&...) { throwError(); } -}; - class FunctionIf : public FunctionIfBase { @@ -193,52 +178,66 @@ private: template ColumnPtr executeRightType( - const ColumnUInt8 * cond_col, - const ColumnsWithTypeAndName & arguments, - const ColVecT0 * col_left) const + [[maybe_unused]] const ColumnUInt8 * cond_col, + [[maybe_unused]] const ColumnsWithTypeAndName & arguments, + [[maybe_unused]] const ColVecT0 * col_left) const { using ResultType = typename NumberTraits::ResultOfIf::Type; - const IColumn * col_right_untyped = arguments[2].column.get(); - UInt32 scale = decimalScale(arguments); - - if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) + if constexpr (std::is_same_v) { - return NumIfImpl::vectorVector( - cond_col->getData(), col_left->getData(), col_right_vec->getData(), scale); + return nullptr; } - else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) + else { - return NumIfImpl::vectorConstant( - cond_col->getData(), col_left->getData(), col_right_const->template getValue(), scale); - } + const IColumn * col_right_untyped = arguments[2].column.get(); + UInt32 scale = decimalScale(arguments); - return nullptr; + if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) + { + return NumIfImpl::vectorVector( + cond_col->getData(), col_left->getData(), col_right_vec->getData(), scale); + } + else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) + { + return NumIfImpl::vectorConstant( + cond_col->getData(), col_left->getData(), col_right_const->template getValue(), scale); + } + + return nullptr; + } } template ColumnPtr executeConstRightType( - const ColumnUInt8 * cond_col, - const ColumnsWithTypeAndName & arguments, - const ColumnConst * col_left) const + [[maybe_unused]] const ColumnUInt8 * cond_col, + [[maybe_unused]] const ColumnsWithTypeAndName & arguments, + [[maybe_unused]] const ColumnConst * col_left) const { using ResultType = typename NumberTraits::ResultOfIf::Type; - const IColumn * col_right_untyped = arguments[2].column.get(); - UInt32 scale = decimalScale(arguments); - - if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) + if constexpr (std::is_same_v) { - return NumIfImpl::constantVector( - cond_col->getData(), col_left->template getValue(), col_right_vec->getData(), scale); + return nullptr; } - else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) + else { - return NumIfImpl::constantConstant( - cond_col->getData(), col_left->template getValue(), col_right_const->template getValue(), scale); - } + const IColumn * col_right_untyped = arguments[2].column.get(); + UInt32 scale = decimalScale(arguments); - return nullptr; + if (const auto * col_right_vec = checkAndGetColumn(col_right_untyped)) + { + return NumIfImpl::constantVector( + cond_col->getData(), col_left->template getValue(), col_right_vec->getData(), scale); + } + else if (const auto * col_right_const = checkAndGetColumnConst(col_right_untyped)) + { + return NumIfImpl::constantConstant( + cond_col->getData(), col_left->template getValue(), col_right_const->template getValue(), scale); + } + + return nullptr; + } } template @@ -249,12 +248,14 @@ private: [[maybe_unused]] const ColumnArray * col_left_array, [[maybe_unused]] size_t input_rows_count) const { - if constexpr (std::is_same_v::Type>) + using ResultType = typename NumberTraits::ResultOfIf::Type; + + if constexpr (std::is_same_v) + { return nullptr; + } else { - using ResultType = typename NumberTraits::ResultOfIf::Type; - const IColumn * col_right_untyped = arguments[2].column.get(); if (const auto * col_right_array = checkAndGetColumn(col_right_untyped)) @@ -291,9 +292,9 @@ private: return res; } - } - return nullptr; + return nullptr; + } } template @@ -304,12 +305,14 @@ private: [[maybe_unused]] const ColumnConst * col_left_const_array, [[maybe_unused]] size_t input_rows_count) const { - if constexpr (std::is_same_v::Type>) + using ResultType = typename NumberTraits::ResultOfIf::Type; + + if constexpr (std::is_same_v) + { return nullptr; + } else { - using ResultType = typename NumberTraits::ResultOfIf::Type; - const IColumn * col_right_untyped = arguments[2].column.get(); if (const auto * col_right_array = checkAndGetColumn(col_right_untyped)) @@ -347,37 +350,34 @@ private: return res; } - } - return nullptr; + return nullptr; + } } template - ColumnPtr executeTyped(const ColumnUInt8 * cond_col, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + ColumnPtr executeTyped( + const ColumnUInt8 * cond_col, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { using ColVecT0 = std::conditional_t, ColumnDecimal, ColumnVector>; using ColVecT1 = std::conditional_t, ColumnDecimal, ColumnVector>; const IColumn * col_left_untyped = arguments[1].column.get(); - bool left_ok = false; ColumnPtr right_column = nullptr; if (const auto * col_left = checkAndGetColumn(col_left_untyped)) { - left_ok = true; right_column = executeRightType(cond_col, arguments, col_left); } else if (const auto * col_const_left = checkAndGetColumnConst(col_left_untyped)) { - left_ok = true; right_column = executeConstRightType(cond_col, arguments, col_const_left); } else if (const auto * col_arr_left = checkAndGetColumn(col_left_untyped)) { if (auto col_arr_left_elems = checkAndGetColumn(&col_arr_left->getData())) { - left_ok = true; right_column = executeRightTypeArray( cond_col, arguments, result_type, col_arr_left, input_rows_count); } @@ -386,20 +386,11 @@ private: { if (checkColumn(&assert_cast(col_const_arr_left->getDataColumn()).getData())) { - left_ok = true; right_column = executeConstRightTypeArray( cond_col, arguments, result_type, col_const_arr_left, input_rows_count); } } - if (!left_ok) - return nullptr; - - const ColumnWithTypeAndName & right_column_typed = arguments[2]; - if (!right_column) - throw Exception("Illegal column " + right_column_typed.column->getName() + " of third argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - return right_column; } @@ -643,7 +634,8 @@ private: return result_column; } - ColumnPtr executeForConstAndNullableCondition(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const + ColumnPtr executeForConstAndNullableCondition( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const { const ColumnWithTypeAndName & arg_cond = arguments[0]; bool cond_is_null = arg_cond.column->onlyNull(); @@ -973,7 +965,8 @@ public: using T0 = typename Types::LeftType; using T1 = typename Types::RightType; - return (res = executeTyped(cond_col, arguments, result_type, input_rows_count)) != nullptr; + res = executeTyped(cond_col, arguments, result_type, input_rows_count); + return res != nullptr; }; TypeIndex left_id = arg_then.type->getTypeId(); From de5cc1f89f8250d7af0f062293e0007ac27973ea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 06:55:49 +0300 Subject: [PATCH 171/491] Add test --- tests/queries/0_stateless/01913_if_int_decimal.reference | 3 +++ tests/queries/0_stateless/01913_if_int_decimal.sql | 1 + 2 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/01913_if_int_decimal.reference create mode 100644 tests/queries/0_stateless/01913_if_int_decimal.sql diff --git a/tests/queries/0_stateless/01913_if_int_decimal.reference b/tests/queries/0_stateless/01913_if_int_decimal.reference new file mode 100644 index 00000000000..c54e91df3e4 --- /dev/null +++ b/tests/queries/0_stateless/01913_if_int_decimal.reference @@ -0,0 +1,3 @@ +2.0000000000 +1.0000000000 +2.0000000000 diff --git a/tests/queries/0_stateless/01913_if_int_decimal.sql b/tests/queries/0_stateless/01913_if_int_decimal.sql new file mode 100644 index 00000000000..83fb4c352c1 --- /dev/null +++ b/tests/queries/0_stateless/01913_if_int_decimal.sql @@ -0,0 +1 @@ +select number % 2 ? materialize(1)::Decimal(18, 10) : 2 FROM numbers(3); From 76c7a0bd4944cfa857dae42060648af03343fa07 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 07:11:17 +0300 Subject: [PATCH 172/491] Add a test for #17964 --- tests/queries/0_stateless/01914_index_bgranvea.reference | 2 ++ tests/queries/0_stateless/01914_index_bgranvea.sql | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/01914_index_bgranvea.reference create mode 100644 tests/queries/0_stateless/01914_index_bgranvea.sql diff --git a/tests/queries/0_stateless/01914_index_bgranvea.reference b/tests/queries/0_stateless/01914_index_bgranvea.reference new file mode 100644 index 00000000000..85e6138dc5d --- /dev/null +++ b/tests/queries/0_stateless/01914_index_bgranvea.reference @@ -0,0 +1,2 @@ +1 1 1 +1 1 1 diff --git a/tests/queries/0_stateless/01914_index_bgranvea.sql b/tests/queries/0_stateless/01914_index_bgranvea.sql new file mode 100644 index 00000000000..817c9571088 --- /dev/null +++ b/tests/queries/0_stateless/01914_index_bgranvea.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test; +create table test (id UInt64,insid UInt64,insidvalue Nullable(UInt64), index insid_idx (insid) type bloom_filter() granularity 1, index insidvalue_idx (insidvalue) type bloom_filter() granularity 1) ENGINE=MergeTree() ORDER BY (insid,id); + +insert into test values(1,1,1),(2,2,2); + +select * from test where insid IN (1) OR insidvalue IN (1); +select * from test where insid IN (1) AND insidvalue IN (1); + +DROP TABLE test; From 798af52c7d721f454f2eeeda5d7a341b1b6871b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 07:22:57 +0300 Subject: [PATCH 173/491] Add a test for #17367 --- .../0_stateless/01915_for_each_crakjie.reference | 2 ++ tests/queries/0_stateless/01915_for_each_crakjie.sql | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/01915_for_each_crakjie.reference create mode 100644 tests/queries/0_stateless/01915_for_each_crakjie.sql diff --git a/tests/queries/0_stateless/01915_for_each_crakjie.reference b/tests/queries/0_stateless/01915_for_each_crakjie.reference new file mode 100644 index 00000000000..905658eb3f7 --- /dev/null +++ b/tests/queries/0_stateless/01915_for_each_crakjie.reference @@ -0,0 +1,2 @@ +b [2,2.2,2.260035] +a [2,2.2,2.260035] diff --git a/tests/queries/0_stateless/01915_for_each_crakjie.sql b/tests/queries/0_stateless/01915_for_each_crakjie.sql new file mode 100644 index 00000000000..f65e08af61b --- /dev/null +++ b/tests/queries/0_stateless/01915_for_each_crakjie.sql @@ -0,0 +1,10 @@ +WITH arrayJoin(['a', 'b']) AS z +SELECT + z, + sumMergeForEach(x) AS x +FROM +( + SELECT sumStateForEach([1., 1.1, 1.1300175]) AS x + FROM remote('127.0.0.{1,2}', system.one) +) +GROUP BY z; From 01b55626d7652725050574c0f6731a1b592a8534 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 15 Jun 2021 07:37:17 +0300 Subject: [PATCH 174/491] Update test --- .../0_stateless/00735_conditional.reference | 24 ++++++++++ .../queries/0_stateless/00735_conditional.sql | 48 +++++++++---------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/tests/queries/0_stateless/00735_conditional.reference b/tests/queries/0_stateless/00735_conditional.reference index 6bee974769d..6308a48218b 100644 --- a/tests/queries/0_stateless/00735_conditional.reference +++ b/tests/queries/0_stateless/00735_conditional.reference @@ -105,6 +105,9 @@ column vs value 0 1 1 Int8 UInt32 Int64 0 1 1 Int8 Float32 Float32 0 1 1 Int8 Float64 Float64 +0 1 1 Int8 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int8 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int8 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int16 Int8 Int16 0 1 1 Int16 Int16 Int16 0 1 1 Int16 Int32 Int32 @@ -114,6 +117,9 @@ column vs value 0 1 1 Int16 UInt32 Int64 0 1 1 Int16 Float32 Float32 0 1 1 Int16 Float64 Float64 +0 1 1 Int16 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int16 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int16 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int32 Int8 Int32 0 1 1 Int32 Int16 Int32 0 1 1 Int32 Int32 Int32 @@ -123,6 +129,9 @@ column vs value 0 1 1 Int32 UInt32 Int64 0 1 1 Int32 Float32 Float64 0 1 1 Int32 Float64 Float64 +0 1 1 Int32 Decimal(9, 0) Decimal(9, 0) +0 1 1 Int32 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int32 Decimal(38, 0) Decimal(38, 0) 0 1 1 Int64 Int8 Int64 0 1 1 Int64 Int16 Int64 0 1 1 Int64 Int32 Int64 @@ -130,6 +139,9 @@ column vs value 0 1 1 Int64 UInt8 Int64 0 1 1 Int64 UInt16 Int64 0 1 1 Int64 UInt32 Int64 +0 1 1 Int64 Decimal(9, 0) Decimal(18, 0) +0 1 1 Int64 Decimal(18, 0) Decimal(18, 0) +0 1 1 Int64 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt8 Int8 Int16 0 1 1 UInt8 Int16 Int16 0 1 1 UInt8 Int32 Int32 @@ -140,6 +152,9 @@ column vs value 0 1 1 UInt8 UInt64 UInt64 0 1 1 UInt8 Float32 Float32 0 1 1 UInt8 Float64 Float64 +0 1 1 UInt8 Decimal(9, 0) Decimal(9, 0) +0 1 1 UInt8 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt8 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt16 Int8 Int32 0 1 1 UInt16 Int16 Int32 0 1 1 UInt16 Int32 Int32 @@ -150,6 +165,9 @@ column vs value 0 1 1 UInt16 UInt64 UInt64 0 1 1 UInt16 Float32 Float32 0 1 1 UInt16 Float64 Float64 +0 1 1 UInt16 Decimal(9, 0) Decimal(9, 0) +0 1 1 UInt16 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt16 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt32 Int8 Int64 0 1 1 UInt32 Int16 Int64 0 1 1 UInt32 Int32 Int64 @@ -160,10 +178,16 @@ column vs value 0 1 1 UInt32 UInt64 UInt64 0 1 1 UInt32 Float32 Float64 0 1 1 UInt32 Float64 Float64 +0 1 1 UInt32 Decimal(9, 0) Decimal(18, 0) +0 1 1 UInt32 Decimal(18, 0) Decimal(18, 0) +0 1 1 UInt32 Decimal(38, 0) Decimal(38, 0) 0 1 1 UInt64 UInt8 UInt64 0 1 1 UInt64 UInt16 UInt64 0 1 1 UInt64 UInt32 UInt64 0 1 1 UInt64 UInt64 UInt64 +0 1 1 UInt64 Decimal(9, 0) Decimal(38, 0) +0 1 1 UInt64 Decimal(18, 0) Decimal(38, 0) +0 1 1 UInt64 Decimal(38, 0) Decimal(38, 0) 1970-01-01 1970-01-02 1970-01-02 Date Date Date 2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime 2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime diff --git a/tests/queries/0_stateless/00735_conditional.sql b/tests/queries/0_stateless/00735_conditional.sql index 04439f4062e..a247ce003b3 100644 --- a/tests/queries/0_stateless/00735_conditional.sql +++ b/tests/queries/0_stateless/00735_conditional.sql @@ -176,9 +176,9 @@ SELECT materialize(toInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -192,9 +192,9 @@ SELECT materialize(toInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -208,9 +208,9 @@ SELECT materialize(toInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt64(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt64(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -224,9 +224,9 @@ SELECT materialize(toInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -240,9 +240,9 @@ SELECT materialize(toUInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -256,9 +256,9 @@ SELECT materialize(toUInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -272,9 +272,9 @@ SELECT materialize(toUInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt64(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } @@ -288,9 +288,9 @@ SELECT materialize(toUInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } +SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toDate(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } From 641f31cc4c58120685df777981e3336572f109ae Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 15 Jun 2021 09:05:54 +0300 Subject: [PATCH 175/491] Skip test 00825_protobuf_format_splitted_nested in case ANTLR. --- tests/queries/skip_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index d5a37b64e88..42a1c14e71e 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -200,6 +200,7 @@ "00825_protobuf_format_nested_optional", "00825_protobuf_format_no_length_delimiter", "00825_protobuf_format_persons", + "00825_protobuf_format_splitted_nested", "00825_protobuf_format_squares", "00825_protobuf_format_table_default", "00826_cross_to_inner_join", From a43c5c0a7f329ae55476fa511410989529ae4145 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 15 Jun 2021 10:27:38 +0300 Subject: [PATCH 176/491] Apply suggestions from code review --- .../sql-reference/aggregate-functions/parametric-functions.md | 2 +- .../sql-reference/aggregate-functions/parametric-functions.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 33df62e5c61..487074eca00 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -536,7 +536,7 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event - `timestamp` — Name of the column containing the timestamp. Data types supported: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) and other unsigned integer types. - `event_column` — Name of the column containing the value of the next event to be returned. Data types supported: [String](../../sql-reference/data-types/string.md) and [Nullable(String)](../../sql-reference/data-types/nullable.md). - `base_condition` — Condition that the base point must fulfill. -- `cond` — Conditions describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). +- `event1`, `event2`, ... — Conditions describing the chain of events. [UInt8](../../sql-reference/data-types/int-uint.md). **Returned values** diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index dc16d139319..cf67dfb54ec 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -525,7 +525,7 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event - `timestamp` — название столбца, содержащего `timestamp`. Поддерживаемые типы данных: [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md#data_type-datetime) и другие беззнаковые целые типы. - `event_column` — название столбца, содержащего значение следующего возвращаемого события. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [Nullable(String)](../../sql-reference/data-types/nullable.md). - `base_condition` — условие, которому должна соответствовать исходная точка. -- `cond` — условия, описывающие цепочку событий. [UInt8](../../sql-reference/data-types/int-uint.md). +- `event1`, `event2`, ... — условия, описывающие цепочку событий. [UInt8](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -538,7 +538,7 @@ sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event Функцию можно использовать, если есть цепочка событий A->B->C->D->E, и вы хотите определить событие, следующее за B->C, то есть D. -Оператор запроса ищет событие после A->B: +Запрос ищет событие после A->B: ``` sql CREATE TABLE test_flow ( From 09ba2af5dc8427f358489cd397bc8f272130771e Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 15 Jun 2021 10:39:04 +0300 Subject: [PATCH 177/491] Update docs/ru/sql-reference/operators/index.md --- docs/ru/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index ca742996a94..7fddf8c56df 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -192,7 +192,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV Вы можете изменить дату, не используя синтаксис `INTERVAL`, а просто добавив или отняв секунды, минуты и часы. Например, чтобы передвинуть дату на один день вперед, можно прибавить к ней значение `60*60*24`. !!! note "Примечание" - Для работы с датами лучше использовать синтаксис `INTERVAL` или функцию `addDays`. Простое сложение (например, синтаксис `now() + ...`) не учитывает региональные настройки времени, например, переход на летнее время. + Синтаксис `INTERVAL` или функция `addDays` предпочтительнее для работы с датами. Сложение с числом (например, синтаксис `now() + ...`) не учитывает региональные настройки времени, например, переход на летнее время. Пример: From e9bfcd20e6db151e8d791aa9b48290aa1b18c616 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 12 Jun 2021 19:45:26 +0300 Subject: [PATCH 178/491] ExpressionJIT fix loop --- src/Core/Settings.h | 2 +- src/Interpreters/JIT/compileFunction.cpp | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2aed174c088..e9379186c0e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -104,7 +104,7 @@ class IColumn; \ M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \ M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \ - M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ + M(UInt64, min_count_to_compile_expression, 0, "The number of identical expressions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 43f914c715a..1500a4049ec 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -75,8 +75,8 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio auto * func = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, function.getName(), module); auto * args = func->args().begin(); - llvm::Value * counter_arg = &*args++; - llvm::Value * columns_arg = &*args++; + llvm::Value * rows_count_arg = args++; + llvm::Value * columns_arg = args++; /// Initialize ColumnDataPlaceholder llvm representation of ColumnData @@ -94,12 +94,14 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio /// Initialize loop + auto * end = llvm::BasicBlock::Create(b.getContext(), "end", func); auto * loop = llvm::BasicBlock::Create(b.getContext(), "loop", func); - b.CreateBr(loop); + b.CreateCondBr(b.CreateICmpEQ(rows_count_arg, llvm::ConstantInt::get(size_type, 0)), end, loop); + b.SetInsertPoint(loop); - auto * counter_phi = b.CreatePHI(counter_arg->getType(), 2); - counter_phi->addIncoming(counter_arg, entry); + auto * counter_phi = b.CreatePHI(rows_count_arg->getType(), 2); + counter_phi->addIncoming(llvm::ConstantInt::get(size_type, 0), entry); for (auto & col : columns) { @@ -158,10 +160,11 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio col.null->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1), cur_block); } - counter_phi->addIncoming(b.CreateSub(counter_phi, llvm::ConstantInt::get(size_type, 1)), cur_block); + auto * value = b.CreateAdd(counter_phi, llvm::ConstantInt::get(size_type, 1)); + counter_phi->addIncoming(value, cur_block); + + b.CreateCondBr(b.CreateICmpEQ(value, rows_count_arg), end, loop); - auto * end = llvm::BasicBlock::Create(b.getContext(), "end", func); - b.CreateCondBr(b.CreateICmpNE(counter_phi, llvm::ConstantInt::get(size_type, 1)), loop, end); b.SetInsertPoint(end); b.CreateRetVoid(); } From a32d9cdbcd48421c0df08e95230a61c57b7f8d42 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 13 Jun 2021 18:14:11 +0300 Subject: [PATCH 179/491] Added compileFunction preudocode documentation --- src/Interpreters/JIT/compileFunction.cpp | 81 ++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 1500a4049ec..384b1a4a781 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -61,7 +61,88 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio /** Algorithm is to create a loop that iterate over ColumnDataRowsSize size_t argument and * over ColumnData data and null_data. On each step compiled expression from function * will be executed over column data and null_data row. + * + * Example of preudocode of generated instructions of function with 1 input column. + * In case of multiple columns more column_i_data, column_i_null_data is created. + * + * void compiled_function(size_t rows_count, ColumnData * columns) + * { + * /// Initialize column values + * + * Column0Type * column_0_data = static_cast(columns[0].data); + * UInt8 * column_0_null_data = static_cast(columns[0].null_data); + * + * /// Initialize other input columns data with indexes < input_columns_count + * + * ResultType * result_column_data = static_cast(columns[input_columns_count].data); + * UInt8 * result_column_null_data = static_cast(columns[input_columns_count].data); + * + * if (rows_count == 0) + * goto end; + * + * /// Loop + * + * size_t counter = 0; + * + * loop: + * + * /// Create column values tuple in case of non nullable type it is just column value + * /// In case of nullable type it is tuple of column value and is column row nullable + * + * Column0Tuple column_0_value; + * if (Column0Type is nullable) + * { + * value[0] = column_0_data; + * value[1] = static_cast(column_1_null_data); + * } + * else + * { + * value[0] = column_0_data + * } + * + * /// Initialize other input column values tuple with indexes < input_columns_count + * /// execute_compiled_expressions function takes input columns values and must return single result value + * + * if (ResultType is nullable) + * { + * (ResultType, bool) result_column_value = execute_compiled_expressions(column_0_value, ...); + * *result_column_data = result_column_value[0]; + * *result_column_null_data = static_cast(result_column_value[1]); + * } + * else + * { + * ResultType result_column_value = execute_compiled_expressions(column_0_value, ...); + * *result_column_data = result_column_value; + * } + * + * /// Increment input and result column current row pointer + * + * ++column_0_data; + * if (Column 0 type is nullable) + * { + * ++column_0_null_data; + * } + * + * ++result_column_data; + * if (ResultType is nullable) + * { + * ++result_column_null_data; + * } + * + * /// Increment loop counter and check if we should exit. + * + * ++counter; + * if (counter == rows_count) + * goto end; + * else + * goto loop; + * + * /// End + * end: + * return; + * } */ + ProfileEvents::increment(ProfileEvents::CompileFunction); const auto & arg_types = function.getArgumentTypes(); From 29f81b235fb684ac3ed6e9b371398eaa6b457f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D1=81=D0=BB=D0=B0=D0=B2=20?= =?UTF-8?q?=D0=A2=D0=B8=D1=85=D0=BE=D0=BD=D0=BE=D0=B2?= <49529856+Slasat@users.noreply.github.com> Date: Tue, 15 Jun 2021 11:09:06 +0300 Subject: [PATCH 180/491] Fixed typos. --- docs/ru/operations/system-tables/trace_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 6d8130c1d00..cc2eb4f9883 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -2,7 +2,7 @@ Содержит экземпляры трассировки стека адресов вызова, собранные с помощью семплирующего профайлера запросов. -ClickHouse создает эту таблицу когда утсановлена настройка [trace_log](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) в конфигурационном файле сервереа. А также настройки [query_profiler_real_time_period_ns](../settings/settings.md#query_profiler_real_time_period_ns) и [query_profiler_cpu_time_period_ns](../settings/settings.md#query_profiler_cpu_time_period_ns). +ClickHouse создает эту таблицу когда установлена настройка [trace_log](../server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) в конфигурационном файле сервера. А также настройки [query_profiler_real_time_period_ns](../settings/settings.md#query_profiler_real_time_period_ns) и [query_profiler_cpu_time_period_ns](../settings/settings.md#query_profiler_cpu_time_period_ns). Для анализа stack traces, используйте функции интроспекции `addressToLine`, `addressToSymbol` и `demangle`. From 4c573b36fe3eb67a9a7d0ebabeaa149f09fa9d8e Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 15 Jun 2021 16:34:53 +0800 Subject: [PATCH 181/491] Fix joinGetOrNull with not-nullable columns --- src/Interpreters/HashJoin.cpp | 54 +++++++++++++------ src/Interpreters/HashJoin.h | 3 +- .../01240_join_get_or_null.reference | 5 ++ .../0_stateless/01240_join_get_or_null.sql | 11 ++++ 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 09c4c538347..6ec021ea290 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -680,7 +680,6 @@ namespace class AddedColumns { public: - struct TypeAndName { DataTypePtr type; @@ -688,24 +687,26 @@ public: String qualified_name; TypeAndName(DataTypePtr type_, const String & name_, const String & qualified_name_) - : type(type_) - , name(name_) - , qualified_name(qualified_name_) - {} + : type(type_), name(name_), qualified_name(qualified_name_) + { + } }; - AddedColumns(const Block & block_with_columns_to_add, - const Block & block, - const Block & saved_block_sample, - const HashJoin & join, - const ColumnRawPtrs & key_columns_, - const Sizes & key_sizes_, - bool is_asof_join) + AddedColumns( + const Block & block_with_columns_to_add, + const Block & block, + const Block & saved_block_sample, + const HashJoin & join, + const ColumnRawPtrs & key_columns_, + const Sizes & key_sizes_, + bool is_asof_join, + bool is_join_get_) : key_columns(key_columns_) , key_sizes(key_sizes_) , rows_to_add(block.rows()) , asof_type(join.getAsofType()) , asof_inequality(join.getAsofInequality()) + , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); if (is_asof_join) @@ -749,8 +750,25 @@ public: if constexpr (has_defaults) applyLazyDefaults(); - for (size_t j = 0, size = right_indexes.size(); j < size; ++j) - columns[j]->insertFrom(*block.getByPosition(right_indexes[j]).column, row_num); + if (is_join_get) + { + /// If it's joinGetOrNull, we need to wrap not-nullable columns in StorageJoin. + for (size_t j = 0, size = right_indexes.size(); j < size; ++j) + { + const auto & column = *block.getByPosition(right_indexes[j]).column; + if (auto * nullable_col = typeid_cast(columns[j].get()); nullable_col && !column.isNullable()) + nullable_col->insertFromNotNullable(column, row_num); + else + columns[j]->insertFrom(column, row_num); + } + } + else + { + for (size_t j = 0, size = right_indexes.size(); j < size; ++j) + { + columns[j]->insertFrom(*block.getByPosition(right_indexes[j]).column, row_num); + } + } } void appendDefaultRow() @@ -787,6 +805,7 @@ private: std::optional asof_type; ASOF::Inequality asof_inequality; const IColumn * left_asof_key = nullptr; + bool is_join_get; void addColumn(const ColumnWithTypeAndName & src_column, const std::string & qualified_name) { @@ -1021,7 +1040,8 @@ void HashJoin::joinBlockImpl( Block & block, const Names & key_names_left, const Block & block_with_columns_to_add, - const Maps & maps_) const + const Maps & maps_, + bool is_join_get) const { constexpr bool is_any_join = STRICTNESS == ASTTableJoin::Strictness::Any; constexpr bool is_all_join = STRICTNESS == ASTTableJoin::Strictness::All; @@ -1065,7 +1085,7 @@ void HashJoin::joinBlockImpl( * For ASOF, the last column is used as the ASOF column */ - AddedColumns added_columns(block_with_columns_to_add, block, savedBlockSample(), *this, left_key_columns, key_sizes, is_asof_join); + AddedColumns added_columns(block_with_columns_to_add, block, savedBlockSample(), *this, left_key_columns, key_sizes, is_asof_join, is_join_get); bool has_required_right_keys = (required_right_keys.columns() != 0); added_columns.need_filter = need_filter || has_required_right_keys; @@ -1274,7 +1294,7 @@ ColumnWithTypeAndName HashJoin::joinGet(const Block & block, const Block & block static_assert(!MapGetter::flagged, "joinGet are not protected from hash table changes between block processing"); joinBlockImpl( - keys, key_names_right, block_with_columns_to_add, std::get(data->maps)); + keys, key_names_right, block_with_columns_to_add, std::get(data->maps), /* is_join_get */ true); return keys.getByPosition(keys.columns() - 1); } diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 5ec8ed92f10..84c447d875e 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -400,7 +400,8 @@ private: Block & block, const Names & key_names_left, const Block & block_with_columns_to_add, - const Maps & maps) const; + const Maps & maps, + bool is_join_get = false) const; void joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const; diff --git a/tests/queries/0_stateless/01240_join_get_or_null.reference b/tests/queries/0_stateless/01240_join_get_or_null.reference index 96e34d5a44c..322d026bea8 100644 --- a/tests/queries/0_stateless/01240_join_get_or_null.reference +++ b/tests/queries/0_stateless/01240_join_get_or_null.reference @@ -1,2 +1,7 @@ \N \N +1396-01-12 +1396-01-13 +\N +\N +\N diff --git a/tests/queries/0_stateless/01240_join_get_or_null.sql b/tests/queries/0_stateless/01240_join_get_or_null.sql index 48fd8228b55..b230cc7bb8d 100644 --- a/tests/queries/0_stateless/01240_join_get_or_null.sql +++ b/tests/queries/0_stateless/01240_join_get_or_null.sql @@ -11,3 +11,14 @@ DROP TABLE join_test; CREATE TABLE join_test (id UInt16, num Array(UInt16)) engine = Join(ANY, LEFT, id); SELECT joinGetOrNull('join_test', 'num', 500); -- { serverError 43 } DROP TABLE join_test; + +drop table if exists test; +create table test (x Date, y String) engine Join(ANY, LEFT, x); +insert into test values ('2017-04-01', '1396-01-12') ,('2017-04-02', '1396-01-13'); + +WITH + A as (SELECT rowNumberInAllBlocks() R, addDays(toDate('2017-04-01'), R) TVV from numbers(5)), + B as (SELECT rowNumberInAllBlocks() R, toDateTime(NULL) TVV from numbers(1)) +SELECT + joinGetOrNull('test', 'y', toDate(A.TVV) ) TV1 +from A LEFT JOIN B USING (R) order by TV1; From b6be838a752a7ba3216505c1a85d6c26bc748e99 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 15 Jun 2021 11:59:02 +0300 Subject: [PATCH 182/491] Add test case to 01049_join_low_card_crash --- tests/queries/0_stateless/01049_join_low_card_crash.reference | 4 ++++ tests/queries/0_stateless/01049_join_low_card_crash.sql | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/queries/0_stateless/01049_join_low_card_crash.reference b/tests/queries/0_stateless/01049_join_low_card_crash.reference index 648a5de0a87..33e6920cd7a 100644 --- a/tests/queries/0_stateless/01049_join_low_card_crash.reference +++ b/tests/queries/0_stateless/01049_join_low_card_crash.reference @@ -4,9 +4,13 @@ a 1 2 b 0 3 0 b 3 a 1 a 2 +0 +7 a 1 b \N a 1 2 b \N 3 a 1 a 2 \N \N b 3 +7 +\N diff --git a/tests/queries/0_stateless/01049_join_low_card_crash.sql b/tests/queries/0_stateless/01049_join_low_card_crash.sql index 4465fc9581f..6681014effa 100644 --- a/tests/queries/0_stateless/01049_join_low_card_crash.sql +++ b/tests/queries/0_stateless/01049_join_low_card_crash.sql @@ -11,11 +11,15 @@ SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) js2 USING (foo) ORDER BY foo; SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; +-- https://github.com/ClickHouse/ClickHouse/issues/20315#issuecomment-789579457 +SELECT materialize(js2.k) FROM (SELECT toLowCardinality(number) AS k FROM numbers(1)) AS js1 FULL OUTER JOIN (SELECT number + 7 AS k FROM numbers(1)) AS js2 USING (k) ORDER BY js2.k; + SET join_use_nulls = 1; SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) js2 USING (foo) ORDER BY foo; SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; +SELECT materialize(js2.k) FROM (SELECT toLowCardinality(number) AS k FROM numbers(1)) AS js1 FULL OUTER JOIN (SELECT number + 7 AS k FROM numbers(1)) AS js2 USING (k) ORDER BY js2.k; DROP TABLE Alpha; DROP TABLE Beta; From b6fc471270d98145ea97d840c13885670851cf5a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 12:52:34 +0300 Subject: [PATCH 183/491] Fix invalid header for (LowCardinality IN (tuple)). --- src/Functions/in.cpp | 31 ++++++++++++++++++- .../0_stateless/01906_lc_in_bug.reference | 2 ++ tests/queries/0_stateless/01906_lc_in_bug.sql | 8 +++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01906_lc_in_bug.reference create mode 100644 tests/queries/0_stateless/01906_lc_in_bug.sql diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 827e0212396..70c669fea28 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -3,10 +3,12 @@ #include #include #include +#include #include #include #include #include +#include #include @@ -67,6 +69,12 @@ public: return 2; } + /// Do not use default implementation for LowCardinality. + /// For now, Set may be const or non const column, depending on how it was created. + /// But we will return UInt8 for any case. + /// TODO: we could use special implementation later. + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override { return std::make_shared(); @@ -122,7 +130,28 @@ public: else columns_of_key_columns.insert(left_arg); - return set->execute(columns_of_key_columns, negative); + ColumnPtr lc_indexes = nullptr; + if (columns_of_key_columns.columns() == 1) + { + auto & arg = columns_of_key_columns.safeGetByPosition(0); + const auto * col = arg.column.get(); + if (const auto * const_col = typeid_cast(col)) + col = &const_col->getDataColumn(); + + if (const auto * lc = typeid_cast(col)) + { + lc_indexes = lc->getIndexesPtr(); + arg.column = lc->getDictionary().getNestedColumn(); + arg.type = removeLowCardinality(arg.type); + } + } + + auto res = set->execute(columns_of_key_columns, negative); + + if (lc_indexes) + return res->index(*lc_indexes, 0); + + return res; } }; diff --git a/tests/queries/0_stateless/01906_lc_in_bug.reference b/tests/queries/0_stateless/01906_lc_in_bug.reference new file mode 100644 index 00000000000..9fe1650abf0 --- /dev/null +++ b/tests/queries/0_stateless/01906_lc_in_bug.reference @@ -0,0 +1,2 @@ +1 0 +3 1 diff --git a/tests/queries/0_stateless/01906_lc_in_bug.sql b/tests/queries/0_stateless/01906_lc_in_bug.sql new file mode 100644 index 00000000000..f8f41da31ae --- /dev/null +++ b/tests/queries/0_stateless/01906_lc_in_bug.sql @@ -0,0 +1,8 @@ +drop table if exists tab; +create table tab (x LowCardinality(String)) engine = MergeTree order by tuple(); + +insert into tab values ('a'), ('bb'), ('a'), ('cc'); + +select count() as c, x in ('a', 'bb') as g from tab group by g order by c; + +drop table if exists tab; From e3ae2f6e7aa58c281bbd87e628fe990ffc833317 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 22 May 2021 01:01:21 +0800 Subject: [PATCH 184/491] Rewrite more alias columns --- src/Interpreters/ColumnAliasesVisitor.cpp | 44 ++++++++----------- src/Interpreters/ColumnAliasesVisitor.h | 21 +++++---- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Interpreters/TreeRewriter.cpp | 7 +-- src/Interpreters/addTypeConversionToAST.cpp | 2 +- .../replaceAliasColumnsInQuery.cpp | 5 ++- src/Interpreters/replaceAliasColumnsInQuery.h | 3 +- src/Storages/ReadInOrderOptimizer.cpp | 5 ++- src/Storages/ReadInOrderOptimizer.h | 1 + .../01576_alias_column_rewrite.reference | 9 ++-- .../01576_alias_column_rewrite.sql | 12 +++++ 11 files changed, 63 insertions(+), 48 deletions(-) diff --git a/src/Interpreters/ColumnAliasesVisitor.cpp b/src/Interpreters/ColumnAliasesVisitor.cpp index f41dbd19047..ab72595a491 100644 --- a/src/Interpreters/ColumnAliasesVisitor.cpp +++ b/src/Interpreters/ColumnAliasesVisitor.cpp @@ -25,35 +25,15 @@ bool ColumnAliasesMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) return !(node->as() || node->as() - || node->as() - || node->as() - || node->as()); + || node->as()); } void ColumnAliasesMatcher::visit(ASTPtr & ast, Data & data) { - // If it's select query, only replace filters. - if (auto * query = ast->as()) - { - if (query->where()) - Visitor(data).visit(query->refWhere()); - if (query->prewhere()) - Visitor(data).visit(query->refPrewhere()); - - return; - } - - if (auto * node = ast->as()) - { - visit(*node, ast, data); - return; - } - - if (auto * node = ast->as()) - { - visit(*node, ast, data); - return; - } + if (auto * func = ast->as()) + visit(*func, ast, data); + else if (auto * ident = ast->as()) + visit(*ident, ast, data); } void ColumnAliasesMatcher::visit(ASTFunction & node, ASTPtr & /*ast*/, Data & data) @@ -81,13 +61,25 @@ void ColumnAliasesMatcher::visit(ASTIdentifier & node, ASTPtr & ast, Data & data { if (auto column_name = IdentifierSemantic::getColumnName(node)) { - if (data.forbidden_columns.count(*column_name) || data.private_aliases.count(*column_name) || !data.columns.has(*column_name)) + if (data.array_join_result_columns.count(*column_name) || data.array_join_source_columns.count(*column_name) + || data.private_aliases.count(*column_name) || !data.columns.has(*column_name)) return; const auto & col = data.columns.get(*column_name); if (col.default_desc.kind == ColumnDefaultKind::Alias) { + auto alias = node.tryGetAlias(); + auto alias_expr = col.default_desc.expression->clone(); + auto original_column = alias_expr->getColumnName(); + // If expanded alias is used in array join, avoid expansion, otherwise the column will be mis-array joined + if (data.array_join_result_columns.count(original_column) || data.array_join_source_columns.count(original_column)) + return; ast = addTypeConversionToAST(col.default_desc.expression->clone(), col.type->getName(), data.columns.getAll(), data.context); + if (!alias.empty()) + ast->setAlias(alias); + else + ast->setAlias(*column_name); + // revisit ast to track recursive alias columns Visitor(data).visit(ast); } diff --git a/src/Interpreters/ColumnAliasesVisitor.h b/src/Interpreters/ColumnAliasesVisitor.h index a1f8e79f64c..e340ab0daa0 100644 --- a/src/Interpreters/ColumnAliasesVisitor.h +++ b/src/Interpreters/ColumnAliasesVisitor.h @@ -52,20 +52,23 @@ public: { const ColumnsDescription & columns; - /// forbidden_columns are from array join, we can't rewrite alias columns involved in array join. - /// Do not analyze joined columns. - /// They may have aliases and come to description as is. - const NameSet & forbidden_columns; + /// columns from array_join_result_to_source cannot be expanded. + NameSet array_join_result_columns; + NameSet array_join_source_columns; ContextPtr context; /// private_aliases are from lambda, so these are local names. NameSet private_aliases; - Data(const ColumnsDescription & columns_, const NameSet & forbidden_columns_, ContextPtr context_) - : columns(columns_) - , forbidden_columns(forbidden_columns_) - , context(context_) - {} + Data(const ColumnsDescription & columns_, const NameToNameMap & array_join_result_columns_, ContextPtr context_) + : columns(columns_), context(context_) + { + for (const auto & [result, source] : array_join_result_columns_) + { + array_join_result_columns.insert(result); + array_join_source_columns.insert(source); + } + } }; static void visit(ASTPtr & ast, Data & data); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 85b9026c642..a7c1202276e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1636,7 +1636,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() column_expr = column_default->expression->clone(); // recursive visit for alias to alias replaceAliasColumnsInQuery( - column_expr, metadata_snapshot->getColumns(), syntax_analyzer_result->getArrayJoinSourceNameSet(), context); + column_expr, metadata_snapshot->getColumns(), syntax_analyzer_result->array_join_result_to_source, context); column_expr = addTypeConversionToAST( std::move(column_expr), column_decl.type->getName(), metadata_snapshot->getColumns().getAll(), context); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 81186a239c9..a8f5389ed10 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -929,14 +929,15 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// array_join_alias_to_name, array_join_result_to_source. getArrayJoinedColumns(query, result, select_query, result.source_columns, source_columns_set); - setJoinStrictness(*select_query, settings.join_default_strictness, settings.any_join_distinct_right_table_keys, - result.analyzed_join->table_join); + setJoinStrictness( + *select_query, settings.join_default_strictness, settings.any_join_distinct_right_table_keys, result.analyzed_join->table_join); + collectJoinedColumns(*result.analyzed_join, *select_query, tables_with_columns, result.aliases); /// rewrite filters for select query, must go after getArrayJoinedColumns if (settings.optimize_respect_aliases && result.metadata_snapshot) { - replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), result.getArrayJoinSourceNameSet(), getContext()); + replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), result.array_join_result_to_source, getContext()); } result.aggregates = getAggregates(query, *select_query); diff --git a/src/Interpreters/addTypeConversionToAST.cpp b/src/Interpreters/addTypeConversionToAST.cpp index 295ac858e28..ba67ec762a9 100644 --- a/src/Interpreters/addTypeConversionToAST.cpp +++ b/src/Interpreters/addTypeConversionToAST.cpp @@ -45,7 +45,7 @@ ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const Nam auto block = actions->getSampleBlock(); - auto desc_type = block.getByName(ast->getColumnName()).type; + auto desc_type = block.getByName(ast->getAliasOrColumnName()).type; if (desc_type->getName() != type_name) return addTypeConversionToAST(std::move(ast), type_name); diff --git a/src/Interpreters/replaceAliasColumnsInQuery.cpp b/src/Interpreters/replaceAliasColumnsInQuery.cpp index 0bc8828c878..3f789ec3d4f 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.cpp +++ b/src/Interpreters/replaceAliasColumnsInQuery.cpp @@ -6,9 +6,10 @@ namespace DB { -void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextPtr context) +void replaceAliasColumnsInQuery( + ASTPtr & ast, const ColumnsDescription & columns, const NameToNameMap & array_join_result_to_source, ContextPtr context) { - ColumnAliasesVisitor::Data aliases_column_data(columns, forbidden_columns, context); + ColumnAliasesVisitor::Data aliases_column_data(columns, array_join_result_to_source, context); ColumnAliasesVisitor aliases_column_visitor(aliases_column_data); aliases_column_visitor.visit(ast); } diff --git a/src/Interpreters/replaceAliasColumnsInQuery.h b/src/Interpreters/replaceAliasColumnsInQuery.h index 92d2686b45b..fadebe3c9e6 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.h +++ b/src/Interpreters/replaceAliasColumnsInQuery.h @@ -10,6 +10,7 @@ namespace DB class ColumnsDescription; -void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, ContextPtr context); +void replaceAliasColumnsInQuery( + ASTPtr & ast, const ColumnsDescription & columns, const NameToNameMap & array_join_result_to_source, ContextPtr context); } diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 3bb7034b588..87273330b34 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -32,6 +32,9 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( /// They may have aliases and come to description as is. /// We can mismatch them with order key columns at stage of fetching columns. forbidden_columns = syntax_result->getArrayJoinSourceNameSet(); + + // array join result columns cannot be used in alias expansion. + array_join_result_to_source = syntax_result->array_join_result_to_source; } InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const @@ -133,7 +136,7 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & if (context->getSettingsRef().optimize_respect_aliases && aliased_columns.contains(required_sort_description[i].column_name)) { auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); - replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), forbidden_columns, context); + replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), array_join_result_to_source, context); auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); const auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context).getActions(true); diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 0af1121db32..0abf2923a98 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -28,6 +28,7 @@ private: /// Actions for every element of order expression to analyze functions for monotonicity ManyExpressionActions elements_actions; NameSet forbidden_columns; + NameToNameMap array_join_result_to_source; SortDescription required_sort_description; }; } diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.reference b/tests/queries/0_stateless/01576_alias_column_rewrite.reference index 9d3db8c1d00..ef598570b10 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.reference +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.reference @@ -26,13 +26,13 @@ Expression (Projection) MergingSorted (Merge sorted streams for ORDER BY) MergeSorting (Merge sorted blocks for ORDER BY) PartialSorting (Sort each block for ORDER BY) - Expression ((Before ORDER BY + Add table aliases)) + Expression (Before ORDER BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromMergeTree Expression (Projection) Limit (preliminary LIMIT) FinishSorting - Expression ((Before ORDER BY + Add table aliases)) + Expression (Before ORDER BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromMergeTree Expression (Projection) @@ -44,12 +44,12 @@ Expression (Projection) optimize_aggregation_in_order Expression ((Projection + Before ORDER BY)) Aggregating - Expression ((Before GROUP BY + Add table aliases)) + Expression (Before GROUP BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromMergeTree Expression ((Projection + Before ORDER BY)) Aggregating - Expression ((Before GROUP BY + Add table aliases)) + Expression (Before GROUP BY) SettingQuotaAndLimits (Set limits and quota after reading from storage) ReadFromMergeTree Expression ((Projection + Before ORDER BY)) @@ -60,3 +60,4 @@ Expression ((Projection + Before ORDER BY)) second-index 1 1 +1 diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql index fabe20010af..cab32db0192 100644 --- a/tests/queries/0_stateless/01576_alias_column_rewrite.sql +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql @@ -115,3 +115,15 @@ SELECT COUNT() == 1 FROM test_index WHERE key_uint32 = 1; SELECT COUNT() == 1 FROM test_index WHERE toUInt32(key_string) = 1; DROP TABLE IF EXISTS test_index; + +-- check alias column can be used to match projections +drop table if exists p; +create table pd (dt DateTime, i int, dt_m DateTime alias toStartOfMinute(dt)) engine Distributed(test_shard_localhost, currentDatabase(), 'pl'); +create table pl (dt DateTime, i int, projection p (select sum(i) group by toStartOfMinute(dt))) engine MergeTree order by dt; + +insert into pl values ('2020-10-24', 1); + +select sum(i) from pd group by dt_m settings allow_experimental_projection_optimization = 1, force_optimize_projection = 1; + +drop table pd; +drop table pl; From 6795209ae4b49750b6a45868c2f1fb6aecffb4c5 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 12 Jun 2021 22:41:50 +0800 Subject: [PATCH 185/491] Reuse local variable --- src/Interpreters/ColumnAliasesVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ColumnAliasesVisitor.cpp b/src/Interpreters/ColumnAliasesVisitor.cpp index ab72595a491..dc445775cbb 100644 --- a/src/Interpreters/ColumnAliasesVisitor.cpp +++ b/src/Interpreters/ColumnAliasesVisitor.cpp @@ -74,7 +74,7 @@ void ColumnAliasesMatcher::visit(ASTIdentifier & node, ASTPtr & ast, Data & data // If expanded alias is used in array join, avoid expansion, otherwise the column will be mis-array joined if (data.array_join_result_columns.count(original_column) || data.array_join_source_columns.count(original_column)) return; - ast = addTypeConversionToAST(col.default_desc.expression->clone(), col.type->getName(), data.columns.getAll(), data.context); + ast = addTypeConversionToAST(std::move(alias_expr), col.type->getName(), data.columns.getAll(), data.context); if (!alias.empty()) ast->setAlias(alias); else From cf2c3ee0d8b5ecdc7fcf467771b5f4ca04cd1f2c Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 15 Jun 2021 12:58:51 +0300 Subject: [PATCH 186/491] Update example for date and time operator Co-authored-by: Nikolai Kochetov --- docs/en/sql-reference/operators/index.md | 11 +++++++---- docs/ru/sql-reference/operators/index.md | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 58ab0068c52..872c98acdf0 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -197,13 +197,16 @@ You can work with dates without using `INTERVAL`, just by adding or subtracting Examples: ``` sql -SELECT now() AS current_date_time, current_date_time + 60*60*96; +SELECT + toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, + time + 60 * 60 * 24 AS time_plus_24_hours, + time + toIntervalDay(1) AS time_plus_1_day ``` ``` text -┌───current_date_time─┬─plus(now(), multiply(multiply(60, 60), 96))─┐ -│ 2021-06-12 17:34:49 │ 2021-06-16 17:34:49 │ -└─────────────────────┴─────────────────────────────────────────────┘ +┌────────────────time─┬──time_plus_24_hours─┬─────time_plus_1_day─┐ +│ 2014-10-26 00:00:00 │ 2014-10-26 23:00:00 │ 2014-10-27 00:00:00 │ +└─────────────────────┴─────────────────────┴─────────────────────┘ ``` **See Also** diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 7fddf8c56df..32501268e5b 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -197,13 +197,16 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV Пример: ``` sql -SELECT now() AS current_date_time, current_date_time + 60*60*96; +SELECT + toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, + time + 60 * 60 * 24 AS time_plus_24_hours, + time + toIntervalDay(1) AS time_plus_1_day ``` ``` text -┌───current_date_time─┬─plus(now(), multiply(multiply(60, 60), 96))─┐ -│ 2021-06-12 17:34:49 │ 2021-06-16 17:34:49 │ -└─────────────────────┴─────────────────────────────────────────────┘ +┌────────────────time─┬──time_plus_24_hours─┬─────time_plus_1_day─┐ +│ 2014-10-26 00:00:00 │ 2014-10-26 23:00:00 │ 2014-10-27 00:00:00 │ +└─────────────────────┴─────────────────────┴─────────────────────┘ ``` **Смотрите также** From 36e8c8216111653a38b65deda7ab90eef3552353 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 12:59:02 +0300 Subject: [PATCH 187/491] Add comment. --- src/Functions/in.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 70c669fea28..3448e09b990 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -130,6 +130,8 @@ public: else columns_of_key_columns.insert(left_arg); + /// Replace single LowCardinality column to it's dictionary if possible. + /// It is good enough optimization, but it does not use LowCardinality cache. ColumnPtr lc_indexes = nullptr; if (columns_of_key_columns.columns() == 1) { From e09a09f7759a0e13e4faf700efa79415ff1c0fdc Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Tue, 15 Jun 2021 13:01:20 +0300 Subject: [PATCH 188/491] Update index.md --- docs/en/sql-reference/operators/index.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 872c98acdf0..268e56a5034 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -197,10 +197,7 @@ You can work with dates without using `INTERVAL`, just by adding or subtracting Examples: ``` sql -SELECT - toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, - time + 60 * 60 * 24 AS time_plus_24_hours, - time + toIntervalDay(1) AS time_plus_1_day +SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 60 * 24 AS time_plus_24_hours, time + toIntervalDay(1) AS time_plus_1_day; ``` ``` text From 1a79540457a75ee18e4cc72b3880468018822d3d Mon Sep 17 00:00:00 2001 From: kirillikoff Date: Tue, 15 Jun 2021 13:01:53 +0300 Subject: [PATCH 189/491] Update index.md --- docs/ru/sql-reference/operators/index.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 32501268e5b..5cf21b64079 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -197,10 +197,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL '4' day + INTERV Пример: ``` sql -SELECT - toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, - time + 60 * 60 * 24 AS time_plus_24_hours, - time + toIntervalDay(1) AS time_plus_1_day +SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 60 * 24 AS time_plus_24_hours, time + toIntervalDay(1) AS time_plus_1_day; ``` ``` text From aaac231c6a87a3e16dc28c94fcad417a01b0f89e Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 15 Jun 2021 18:28:13 +0800 Subject: [PATCH 190/491] Record expanded aliases for RBAC --- src/Interpreters/ColumnAliasesVisitor.cpp | 1 + src/Interpreters/ColumnAliasesVisitor.h | 3 +++ src/Interpreters/InterpreterSelectQuery.cpp | 8 ++++++-- src/Interpreters/TreeRewriter.cpp | 3 ++- src/Interpreters/TreeRewriter.h | 4 ++++ src/Interpreters/replaceAliasColumnsInQuery.cpp | 3 ++- src/Interpreters/replaceAliasColumnsInQuery.h | 2 +- 7 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ColumnAliasesVisitor.cpp b/src/Interpreters/ColumnAliasesVisitor.cpp index dc445775cbb..f90a3967df3 100644 --- a/src/Interpreters/ColumnAliasesVisitor.cpp +++ b/src/Interpreters/ColumnAliasesVisitor.cpp @@ -74,6 +74,7 @@ void ColumnAliasesMatcher::visit(ASTIdentifier & node, ASTPtr & ast, Data & data // If expanded alias is used in array join, avoid expansion, otherwise the column will be mis-array joined if (data.array_join_result_columns.count(original_column) || data.array_join_source_columns.count(original_column)) return; + data.expanded_aliases.insert(*column_name); ast = addTypeConversionToAST(std::move(alias_expr), col.type->getName(), data.columns.getAll(), data.context); if (!alias.empty()) ast->setAlias(alias); diff --git a/src/Interpreters/ColumnAliasesVisitor.h b/src/Interpreters/ColumnAliasesVisitor.h index e340ab0daa0..3d720455b30 100644 --- a/src/Interpreters/ColumnAliasesVisitor.h +++ b/src/Interpreters/ColumnAliasesVisitor.h @@ -60,6 +60,9 @@ public: /// private_aliases are from lambda, so these are local names. NameSet private_aliases; + /// expanded_aliases are expanded to their alias expressions; + NameSet expanded_aliases; + Data(const ColumnsDescription & columns_, const NameToNameMap & array_join_result_columns_, ContextPtr context_) : columns(columns_), context(context_) { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a7c1202276e..b43e3f84d64 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -251,6 +251,9 @@ static void checkAccessRightsForSelect( /// General check. context->checkAccess(AccessType::SELECT, table_id, required_columns); + + /// Also check if expanded aliases can be accessed + context->checkAccess(AccessType::SELECT, table_id, syntax_analyzer_result.getExpandedAliases()); } /// Returns true if we should ignore quotas and limits for a specified table in the system database. @@ -545,8 +548,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (table_id && got_storage_from_query && !joined_tables.isLeftTableFunction()) { - /// The current user should have the SELECT privilege. - /// If this table_id is for a table function we don't check access rights here because in this case they have been already checked in ITableFunction::execute(). + /// The current user should have the SELECT privilege. If this table_id is for a table + /// function we don't check access rights here because in this case they have been already + /// checked in ITableFunction::execute(). checkAccessRightsForSelect(context, table_id, metadata_snapshot, required_columns, *syntax_analyzer_result); /// Remove limits for some tables in the `system` database. diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index a8f5389ed10..61c7363e1ef 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -937,7 +937,8 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// rewrite filters for select query, must go after getArrayJoinedColumns if (settings.optimize_respect_aliases && result.metadata_snapshot) { - replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), result.array_join_result_to_source, getContext()); + result.expanded_aliases + = replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), result.array_join_result_to_source, getContext()); } result.aggregates = getAggregates(query, *select_query); diff --git a/src/Interpreters/TreeRewriter.h b/src/Interpreters/TreeRewriter.h index 64a02cd48a0..52254cd4219 100644 --- a/src/Interpreters/TreeRewriter.h +++ b/src/Interpreters/TreeRewriter.h @@ -32,6 +32,9 @@ struct TreeRewriterResult /// Set of columns that are enough to read from the table to evaluate the expression. It does not include joined columns. NamesAndTypesList required_source_columns; + /// Set of alias columns that are expanded to their alias expressions. We still need the original columns to check access permission. + NameSet expanded_aliases; + Aliases aliases; std::vector aggregates; @@ -78,6 +81,7 @@ struct TreeRewriterResult void collectUsedColumns(const ASTPtr & query, bool is_select); Names requiredSourceColumns() const { return required_source_columns.getNames(); } NameSet getArrayJoinSourceNameSet() const; + Names getExpandedAliases() const { return {expanded_aliases.begin(), expanded_aliases.end()}; } const Scalars & getScalars() const { return scalars; } }; diff --git a/src/Interpreters/replaceAliasColumnsInQuery.cpp b/src/Interpreters/replaceAliasColumnsInQuery.cpp index 3f789ec3d4f..8e20df518c4 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.cpp +++ b/src/Interpreters/replaceAliasColumnsInQuery.cpp @@ -6,12 +6,13 @@ namespace DB { -void replaceAliasColumnsInQuery( +NameSet replaceAliasColumnsInQuery( ASTPtr & ast, const ColumnsDescription & columns, const NameToNameMap & array_join_result_to_source, ContextPtr context) { ColumnAliasesVisitor::Data aliases_column_data(columns, array_join_result_to_source, context); ColumnAliasesVisitor aliases_column_visitor(aliases_column_data); aliases_column_visitor.visit(ast); + return aliases_column_data.expanded_aliases; } } diff --git a/src/Interpreters/replaceAliasColumnsInQuery.h b/src/Interpreters/replaceAliasColumnsInQuery.h index fadebe3c9e6..8c54123f34c 100644 --- a/src/Interpreters/replaceAliasColumnsInQuery.h +++ b/src/Interpreters/replaceAliasColumnsInQuery.h @@ -10,7 +10,7 @@ namespace DB class ColumnsDescription; -void replaceAliasColumnsInQuery( +NameSet replaceAliasColumnsInQuery( ASTPtr & ast, const ColumnsDescription & columns, const NameToNameMap & array_join_result_to_source, ContextPtr context); } From 8bf897b9c69f5672433f94b9d94be167c64e296b Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 15 Jun 2021 18:37:22 +0800 Subject: [PATCH 191/491] Add some comment --- src/Interpreters/ColumnAliasesVisitor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/ColumnAliasesVisitor.cpp b/src/Interpreters/ColumnAliasesVisitor.cpp index f90a3967df3..704d3815c67 100644 --- a/src/Interpreters/ColumnAliasesVisitor.cpp +++ b/src/Interpreters/ColumnAliasesVisitor.cpp @@ -76,6 +76,7 @@ void ColumnAliasesMatcher::visit(ASTIdentifier & node, ASTPtr & ast, Data & data return; data.expanded_aliases.insert(*column_name); ast = addTypeConversionToAST(std::move(alias_expr), col.type->getName(), data.columns.getAll(), data.context); + // We need to set back the original column name, or else the process of naming resolution will complain. if (!alias.empty()) ast->setAlias(alias); else From b65c121a451fe648ed953013b02a872cb091c6c7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 13:38:11 +0300 Subject: [PATCH 192/491] Update test_detach_part_wrong_partition_id.py --- .../test_detach_part_wrong_partition_id.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py index 5d41d5c394a..a4f976cc62d 100644 --- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -4,7 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) # Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. -node1 = cluster.add_instance('node1', image='yandex/clickhouse-server', tag='21.6.3.14', stay_alive=True, with_installed_binary=True) +node_21_6 = cluster.add_instance('node_21_6', image='yandex/clickhouse-server', tag='21.6.3.14', stay_alive=True, with_installed_binary=True) @pytest.fixture(scope="module") @@ -19,14 +19,14 @@ def start_cluster(): def test_detach_part_wrong_partition_id(start_cluster): # Here we create table with partition by UUID. - node1.query("create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()") - node1.query("insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)") + node_21_6.query("create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()") + node_21_6.query("insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)") # After restart, partition id will be different. # There is a single 0-level part, which will become broken. # We expect that it will not be removed (as usual for 0-level broken parts), # but moved to /detached - node1.restart_with_latest_version() + node_21_6.restart_with_latest_version() - num_detached = node1.query("select count() from system.detached_parts") + num_detached = node_21_6.query("select count() from system.detached_parts") assert num_detached == '1\n' From c5d7cdd983353fc609d12d8beff4cb724239aa00 Mon Sep 17 00:00:00 2001 From: Tiaonmmn Date: Tue, 15 Jun 2021 19:09:36 +0800 Subject: [PATCH 193/491] Update formats.md A lot of modifications on Chinese translation. --- docs/zh/interfaces/formats.md | 552 +++++++++++++++++++++------------- 1 file changed, 345 insertions(+), 207 deletions(-) diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index 7537166a0bd..ef18679af12 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -5,11 +5,12 @@ toc_title: 输入/输出格式 # 输入/输出格式 {#formats} -ClickHouse可以接受和返回各种格式的数据。输入支持的格式可以用来解析提供给`INSERT`的数据，可以从文件备份表(如File, URL或HDFS)执行`SELECT`，或者读取外部字典。输出支持的格式可用于获取`SELECT`的结果，并支持执行`INSERT`文件的表中。 +ClickHouse可以接受和返回各种格式的数据。受支持的输入格式可用于提交给`INSERT`语句、从文件表(File,URL,HDFS或者外部目录)执行`SELECT`语句，受支持的输出格式可用于格式化`SELECT`语句的返回结果，或者通过`INSERT`写入到文件表。 + 以下是支持的格式: -| 格式 | 输入 | 输出 | +| 格式 | 输入 | 输出 | |-----------------------------------------------------------------------------------------|-------|--------| | [TabSeparated](#tabseparated) | ✔ | ✔ | | [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | @@ -30,8 +31,8 @@ ClickHouse可以接受和返回各种格式的数据。输入支持的格式可 | [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | -| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ | -| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ | +| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | +| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | | [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | | [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | | [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | @@ -49,7 +50,7 @@ ClickHouse可以接受和返回各种格式的数据。输入支持的格式可 | [Parquet](#data-format-parquet) | ✔ | ✔ | | [Arrow](#data-format-arrow) | ✔ | ✔ | | [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | +| [ORC](#data-format-orc) | ✔ | ✔ | | [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [Native](#native) | ✔ | ✔ | @@ -57,21 +58,25 @@ ClickHouse可以接受和返回各种格式的数据。输入支持的格式可 | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✗ | | [LineAsString](#lineasstring) | ✔ | ✗ | +| [Regexp](#data-format-regexp) | ✔ | ✗ | +| [RawBLOB](#rawblob) | ✔ | ✔ | -您可以使用ClickHouse设置控制一些格式处理参数。更多详情设置请参考[设置](../operations/settings/settings.md) + +您可以使用ClickHouse设置一些格式化参数。更多详情设置请参考[设置](../operations/settings/settings.md) ## TabSeparated {#tabseparated} -在TabSeparated分隔格式中，数据按行写入。每行包含由制表符分隔的值。每个值后跟一个制表符，除了行中最后一个值后跟换行。在任何地方都采用严格的Unix换行。最后一行还必须在末尾包含换行。值以文本格式编写，不包含引号，并使用转义的特殊字符。 +在TabSeparated分隔格式中，数据按行写入。每行包含由制表符分隔的值，每个值后跟一个制表符，除了行中最后一个值，最后的值后面是一个换行符。在任何地方都采用严格的Unix换行(\n)。最后一行结束后必须再插入一个换行符。值以文本格式编写，不包含引号，并使用转义的特殊字符。 -这种格式也可以用`TSV`来表示。 +这种格式也被称为`TSV`。 -`TabSeparated`格式便于使用自定义程序和脚本处理数据。默认情况下，它在HTTP接口和命令行客户端的批处理模式中使用。这种格式还允许在不同dbms之间传输数据。例如，您可以从MySQL获取转储并将其上传到ClickHouse，反之亦然。 +`TabSeparated`格式便于其他的程序和脚本处理数据。默认情况下，HTTP接口和命令行客户端的批处理模式中会使用这个格式。这种格式还允许在不同dbms之间传输数据。例如，您可以从MySQL获取转储并将其上传到ClickHouse，反之亦然。 -`TabSeparated`格式支持输出total值(与TOTALS一起使用时)和extreme值(当`extreme`被设置为1时)。在这种情况下，total值和extreme值会在主数据后输出。主要结果、总值和极值之间用空行分隔。示例: +`TabSeparated`格式支持输出总计的结果(当SQL语句包含`WITH TOTALS`)和极值(当`extremes`被设置为1时)。在这种情况下，总计值和极值会在主数据后输出。主要结果、总值和极值之间用空行分隔。示例: ``` sql -SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` +set extremes=1; +SELECT EventDate, count() AS c FROM test.hits_v1 GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated; ``` ``` text @@ -83,7 +88,7 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD 2014-03-22 1031592 2014-03-23 1046491 -1970-01-01 8873898 +0000-00-00 8873898 2014-03-17 1031592 2014-03-23 1406958 @@ -91,39 +96,41 @@ SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORD ### 数据格式化 {#data-formatting} -整数是用十进制形式写的。数字可以在开头包含一个额外的`+`字符(解析时忽略，格式化时不记录)。非负数不能包含负号。在读取时，允许将空字符串解析为零，或者(对于有符号类型)将仅由一个负号组成的字符串解析为零。不符合相应数据类型的数字可以被解析为不同的数字，而不会出现错误消息。 +整数是用十进制形式写的。数字可以在开头包含一个额外的`+`字符(解析时忽略该符号，格式化时不记录该符号)。非负数不能包含负号。在读取时，允许将空字符串解析为零，或者(对于有符号类型)将'-'(仅有减号的字符串)解析为零。不符合相应数据类型的数字可能被解析为数值，而不会出现错误消息。 -浮点数以十进制形式书写。`.`号用作十进制分隔符。支持指数符号，如`inf`、`+inf`、`-inf`和`nan`。浮点数的条目可以以小数点开始或结束。 -在格式化期间，浮点数可能会丢失准确性。 +浮点数以十进制形式书写。用`.`作为小数点的符号。支持指数符号，如`inf`、`+inf`、`-inf`和`nan`。小数点前或后可以不出现数字(如123.或.123)。 +在格式化期间，浮点数精度可能会丢失。 在解析期间，并不严格要求读取与机器可以表示的最接近的数值。 -日期以YYYY-MM-DD格式编写，并以相同的格式解析，但使用任何字符作为分隔符。 -日期和时间以`YYYY-MM-DD hh:mm:ss`的格式书写，并以相同的格式解析，但使用任何字符作为分隔符。 -这一切都发生在客户端或服务器启动时的系统时区(取决于它们对数据的格式)。对于带有时间的日期，夏时制时间未指定。因此，如果转储在夏令时有时间，则转储不会明确地与数据匹配，解析将选择这两次中的一次。 -在读取操作期间，不正确的日期和具有时间的日期可以使用自然溢出或null日期和时间进行分析，而不会出现错误消息。 +日期以`YYYY-MM-DD`格式编写，并以相同的格式解析，但允许使用任何字符作为分隔符。 +日期和时间以`YYYY-MM-DD hh:mm:ss`的格式书写，并以相同的格式解析，但允许使用任何字符作为分隔符。 +时区采用客户端或服务器端时区(取决于谁对数据进行格式化)。对于带有时间的日期，没有是哦用夏时制时间。因此，如果导出的数据采用了夏时制，则实际入库的时间不一定与预期的时间对应，解析将根据解析动作发起方选择时间。 +在读取操作期间，不正确的日期和具有时间的日期可以自然溢出(如2021-01-32)或设置成空日期和时间，而不会出现错误消息。 -有个例外情况，Unix时间戳格式也支持用时间解析日期(如果它恰好由10个十进制数字组成)。其结果与时间区域无关。格式`YYYY-MM-DD hh:mm:ss`和`NNNNNNNNNN`是自动区分的。 +有个例外情况，时间解析也支持Unix时间戳(如果它恰好由10个十进制数字组成)。其结果与时区无关。格式`YYYY-MM-DD hh:mm:ss`和`NNNNNNNNNN`这两种格式会自动转换。 -字符串以反斜杠转义的特殊字符输出。下面的转义序列用于输出:`\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`。解析还支持`\a`、`\v`和`\xHH`(十六进制转义字符)和任何`\c`字符，其中`c`是任何字符(这些序列被转换为`c`)。因此，读取数据支持这样一种格式，即可以将换行符写成`\n`或`\`，或者写成换行符。例如，字符串`Hello world`在单词之间有换行符，而不是空格，可以用以下语法进行解析: +字符串输出时，特殊字符会自动转义。以下转义序列用于输出:`\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\'`, `\\`。解析还支持`\a`、`\v`和`\xHH`(HH代表十六进制编码)和`\c`，其中`c`是任何字符(这些序列被转换为`c`)。因此，读取数据时，换行符可以写成`\n`或`\`。例如，如果要表示字符串`Hello world`中'Hello'与'world'中间的空格实际上是个换行符，可以写成下面的形式: ``` text Hello\nworld - +``` +等同于 +``` text Hello\ world ``` -第二种形式是支持的，因为MySQL读取tab-separated格式数据集的时候也会使用它。 +第二种形式也受支持，因为MySQL导出tab-separated格式的数据时使用这种格式。 -在TabSeparated分隔格式传递数据时需要转义的最小字符集:`Tab`、换行符(LF)和反斜杠。 +使用TabSeparated格式传递数据时至少需要转义以下特殊字符:制表符(\t)、换行符(\n)和反斜杠(\\)。 -只有一小部分符号被转义。您可以很容易地找到一个字符串值，而您的终端将在输出中不显示它。 +只有一小部分符号被转义。您可以很容易地找到一个能够破坏命令行终端输出的特殊字符。 -数组写在方括号内的逗号分隔值列表中。数组中的数字项按正常格式进行格式化。`Date`和`DateTime`类型用单引号表示。字符串使用与上面相同的转义规则在单引号中编写。 +数组用方括号包裹、逗号分隔的形式表示(例如`[11,22,33]`)。数组中的数字项按上述规则进行格式化。`日期`和`日期时间`类型用单引号包裹。字符串用单引号包裹，遵循上述转义规则。 -[NULL](../sql-reference/syntax.md)将输出为`\N`。 +[NULL](https://clickhouse.tech/docs/zh/sql-reference/syntax/)将输出为`\N`。 -[Nested](../sql-reference/data-types/nested-data-structures/nested.md)结构的每个元素都表示为数组。 +[Nested](https://clickhouse.tech/docs/zh/sql-reference/data-types/nested-data-structures/nested/)结构的每个元素都表示为数组。 示例： @@ -153,45 +160,45 @@ SELECT * FROM nestedt FORMAT TSV ## TabSeparatedRaw {#tabseparatedraw} -与`TabSeparated`格式的不同之处在于，写入的行没有转义。 -使用这种格式解析时，每个字段中不允许使用制表符或换行符。 +与`TabSeparated`格式的不同之处在于，写入的数据不会进行转义处理。 +使用这种格式解析时，每个字段中不允许出现制表符或换行符。 -这种格式也可以使用名称`TSVRaw`来表示。 +这种格式也被称为`TSVRaw`。 ## TabSeparatedWithNames {#tabseparatedwithnames} -与`TabSeparated`格式不同的是列名写在第一行。 -在解析过程中，第一行被完全忽略。不能使用列名来确定它们的位置或检查它们的正确性。 +不同于`TabSeparated`，列名会写在第一行。 +在解析过程中，第一行被完全忽略。您不能依赖列名来确定它们的位置或检查它们的正确性。 (将来可能会添加对头行解析的支持。) -这种格式也可以使用名称`TSVWithNames`来表示。 +这种格式也被称为`TSVWithNames`。 ## TabSeparatedWithNamesAndTypes {#tabseparatedwithnamesandtypes} 与`TabSeparated`格式不同的是列名写在第一行，而列类型写在第二行。 在解析过程中，将完全忽略第一行和第二行。 -这种格式也可以使用名称`TSVWithNamesAndTypes`来表示。 +这种格式也被称为`TSVWithNamesAndTypes`。 ## Template {#format-template} 此格式允许指定带有占位符的自定义格式字符串，这些占位符用于指定转义规则。 -它使用设置`format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter`以及其他格式的一些设置(例如转义`JSON`时使用`output_format_json_quote_64bit_integers`) +它使用`format_schema`, `format_schema_rows`, `format_schema_rows_between_delimiter`以及其他格式的一些设置(例如转义`JSON`时使用`output_format_json_quote_64bit_integers`，具体请向下阅读) -设置`format_template_row`指定文件的路径，该文件包含以下语法的行格式字符串: +设置`format_template_row`用于指定行格式文件的路径，该格式文件包含行格式字符串，语法如下: -`delimiter_1${column_1:serializeAs_1}delimiter_2${column_2:serializeAs_2} ... delimiter_N`, +`delimiter_i${column_i:serializeAs_i}delimiter_i${column_i:serializeAs_i} ... delimiter_i`, -其中，`delimiter_i`是值之间的分隔符(`$`符号可以转义为`$$`)， -`column_i`是要选择或插入其值的列的名称或索引(如果为空，则跳过该列)， +其中，`delimiter_i`是各值之间的分隔符(`$`符号可以转义为`$$`)， +`column_i`是选择或插入值的列的名称或索引(如果为空，则跳过该列)， `serializeAs_i`是列值的转义规则。支持以下转义规则: - `CSV`, `JSON`, `XML` (类似于相同名称的格式) - `Escaped` (类似于`TSV`) - `Quoted` (类似于`Values`) -- `Raw` (类似于`TSVRaw`) -- `None` +- `Raw` (不转义，类似于`TSVRaw`) +- `None` (不转义，具体请向下阅读) 如果省略了转义规则，那么将使用`None`。`XML`和`Raw`只适用于输出。 @@ -199,26 +206,26 @@ SELECT * FROM nestedt FORMAT TSV `Search phrase: ${SearchPhrase:Quoted}, count: ${c:Escaped}, ad price: $$${price:JSON};` -`SearchPhrase`、`c`和`price`列的值被转义为`quotation`、`Escaped`和`JSON`将分别在`Search phrase:`， `， count: `， `， ad price: $`和`;`分隔符之间打印(用于选择)或expected(用于插入)。例如: +`SearchPhrase`、`c`和`price`列的值遵循`Quoted`、`Escaped`和`JSON`转义规则，将分别在`Search phrase:`， `， count: `， `， ad price: $`和`;`分隔符之间打印(用于`SELECT`)或输入期望的值(用于`INSERT`)。例如: `Search phrase: 'bathroom interior design', count: 2166, ad price: $3;` -`format_template_rows_between_delimiter`设置指定行之间的分隔符，它将打印(或expected)在每一行之后，最后一行除外(默认为`\n`)。 +`format_template_rows_between_delimiter`设置指定行之间的分隔符，它将打印(或输入期望值)在每一行之后，最后一行除外(该设置默认值为`\n`)。 -设置`format_template_resultset`指定文件路径，该文件包含resultset的格式字符串。resultset的格式字符串与row的格式字符串具有相同的语法，允许指定前缀、后缀和打印一些附加信息的方法。它包含以下占位符而不是列名: +设置`format_template_resultset`指定结果集格式文件路径，该文件包含结果集的格式字符串。结果集的格式字符串与上述的行格式字符串具有相同的语法，并允许指定前缀、后缀，还提供打印一些附加信息的方法。该文件使用如下占位符，用于取代行格式字符串的列名的位置(即`column_i`): -- `data` `format_template_row`格式的数据行，由`format_template_rows_between_delimiter`分隔。此占位符必须是格式字符串中的第一个占位符。 -- `totals` `format_template_row`格式的总值(和WITH TOTALS一起使用) -- `min` `format_template_row`格式的最小值(当极值设置为1时) -- `max` `format_template_row`格式的最大值(当极值设置为1时) -- `rows` 输出行的总数 -- `rows_before_limit` 没有LIMIT的最小行数。仅当查询包含LIMIT时输出。如果查询包含GROUP BY，那么rows_before_limit_at_least就是没有LIMIT的确切行数。 -- `time` 请求执行时间（秒） -- `rows_read` 已读取的行数 -- `bytes_read` 已读取（未压缩）的字节数 +- `data` 代表遵循`format_template_row`格式的数据行，由`format_template_rows_between_delimiter`设置制定的字符分隔。此占位符必须是格式字符串中的第一个占位符。 +- `totals` 代表遵循`format_template_row`格式的数据行，该行用于代表结果的总计值(当SQL语句包含了`WITH TOTALS`) +- `min` 代表遵循`format_template_row`格式的数据行，该行用于代表结果的最小值(当`extremes`设置为1时) +- `max` 代表遵循`format_template_row`格式的数据行，该行用于代表结果的最大值(当`extremes`设置为1时) +- `rows` 代表输出行的总数 +- `rows_before_limit` 代表没有LIMIT限制的结果最小行数。仅当查询包含LIMIT时才输出此值。如果查询包含GROUP BY，那么`rows_before_limit_at_least`就是没有LIMIT的确切行数。 +- `time` 代表请求执行时间(秒) +- `rows_read` 代表已读取的行数 +- `bytes_read` 代表已读取(未压缩)的字节数 -占位符`data`、`totals`、`min`和`max`必须没有指定转义规则(或者必须显式指定`None`)。其余占位符可以指定任何转义规则。 -如果`format_template_resultset`设置为空字符串，则使用`${data}`作为默认值。 +占位符`data`、`totals`、`min`和`max`不允许指定转义规则(允许显式指定`None`)。其余占位符可以指定任何转义规则。 +如果`format_template_resultset`设置为空，则使用`${data}`作为默认值。 对于insert查询，格式允许跳过某些列或某些字段的前缀或后缀(参见示例)。 Select示例: @@ -300,13 +307,13 @@ Some header\n${data}\nTotal rows: ${:CSV}\n Page views: ${PageViews:CSV}, User id: ${UserID:CSV}, Useless field: ${:CSV}, Duration: ${Duration:CSV}, Sign: ${Sign:CSV} ``` -`PageViews`, `UserID`, `Duration`和`Sign` 内部占位符是表中列的名称。将忽略行中`Useless field`后面和后缀中`\nTotal rows:`之后的值。 +`PageViews`, `UserID`, `Duration`和`Sign` 占位符是表中列的名称。将忽略行中`Useless field`后面和后缀中`\nTotal rows:`之后的值。 输入数据中的所有分隔符必须严格等于指定格式字符串中的分隔符。 ## TemplateIgnoreSpaces {#templateignorespaces} 这种格式只适用于输入。 -类似于`Template`，但跳过输入流中分隔符和值之间的空白字符。但是，如果格式字符串包含空格字符，这些字符将会出现在输入流中。还允许指定空占位符(`${}`或`${:None}`)来将一些分隔符分割为单独的部分，以忽略它们之间的空格。这种占位符仅用于跳过空白字符。 +类似于`Template`，但跳过输入流中分隔符和值之间的空白字符。但是，如果格式字符串包含空格字符，这些字符将会出现在输入流中。也允许指定空占位符(`${}`或`${:None}`)来将一些分隔符分割为单独的部分，以忽略它们之间的空格。这种占位符仅用于跳过空白字符。 如果列的值在所有行的顺序相同，那么可以使用这种格式读取`JSON`。可以使用以下请求从格式为[JSON](#json)的输出示例中插入数据： ``` sql @@ -328,7 +335,7 @@ format_template_resultset = '/some/path/resultset.format', format_template_row = ## TSKV {#tskv} -类似于TabSeparated，但是输出的值是name=value格式。名称的转义方式与TabSeparated格式相同，=符号也是转义的。 +类似于TabSeparated，但是输出的值是name=value格式。名称的转义方式与TabSeparated格式相同，=符号也会被转义。 ``` text SearchPhrase= count()=8267016 @@ -343,7 +350,7 @@ SearchPhrase=curtain designs count()=1064 SearchPhrase=baku count()=1000 ``` -[NULL](../sql-reference/syntax.md)格式为`\N`。 +[NULL](../sql-reference/syntax.md)转化为`\N`。 ``` sql SELECT * FROM t_null FORMAT TSKV @@ -353,28 +360,29 @@ SELECT * FROM t_null FORMAT TSKV x=1 y=\N ``` -当有大量的小列时，这种格式是无效的，并且通常没有理由使用它。不过，就效率而言，它并不比JSONEachRow差。 -这种格式支持数据输出和解析。对于解析，不同列的值支持任何顺序。省略某些值是可以接受的——它们被视为与其默认值相等。在这种情况下，0和空白行被用作默认值。不支持在表中指定的复杂值作为缺省值。 +当有大量的小列时，这种格式效率十分低下，并且通常没有理由使用它。不过，就效率而言，它并不比JSONEachRow差。 +这种格式支持数据输出和解析。用于解析时，可以任意指定列的顺序，也可以省略某些列，那些列的值为该列的默认值，一般情况下为0或空白。不支持将可在表中可指定的复杂值设为默认值。 -解析允许存在不带等号或值的附加字段`tskv`。此字段被忽略。 +解析时允许出现后没有=的字段`tskv`。此字段会被忽略。 ## CSV {#csv} 按`,`分隔的数据格式([RFC](https://tools.ietf.org/html/rfc4180))。 -格式化时，行是用双引号括起来的。字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。日期和时间也会以双引号包括。数字的输出不带引号。值由一个单独的字符隔开，这个字符默认是`,`。行使用Unix换行符（LF）分隔。数组序列化成CSV规则如下：首先将数组序列化为`TabSeparated`格式的字符串，然后将结果字符串用双引号包括输出到`CSV`。`CSV`格式的元组被序列化为单独的列（即它们在元组中的嵌套关系会丢失）。 +格式化时，每一行的值会用双引号括起，日期和时间也会以双引号包括。数字不用双引号括起，字符串中的双引号会以两个双引号输出，除此之外没有其他规则来做字符转义了。值由分隔符隔开，这个分隔符默认是`,`。每一行使用Unix换行符(LF,\n)分隔。 +数组序列化成CSV规则如下：首先将数组序列化为`TabSeparated`格式的字符串，然后将结果字符串用双引号括起后输出到`CSV`。`CSV`格式的元组被序列化为单独的列(即它们在元组中的嵌套关系会丢失)。 ``` bash $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv ``` -\* 默认情况下间隔符是`,` ，在[format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter)中可以了解更多分隔符配置。 +\* 默认情况下分隔符是`,` ，在[format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter)中可以了解更多分隔符配置。 -解析的时候，可以使用或不使用引号来解析所有值。支持双引号和单引号。行也可以不用引号排列。在这种情况下，它们被解析为逗号或换行符（`CR或`LF`）。在解析不带引号的行时，若违反`RFC`规则，会忽略前缀和结尾的空格和制表符。对于换行，全部支持Unix（LF），Windows（CR LF）和Mac OS Classic（CR LF）。 +解析的时候，值可以使用或不使用双引号或者单引号括起来。在这种情况下，每行通过分隔符或换行符(`CR`或`LF`)区分。违反`RFC`规则的是，在解析未用引号括起的行时，会忽略前缀和结尾的空格和制表符。对于换行符，Unix(LF,\n)，Windows(CR LF\r\n)和Mac OS Classic(CR LF\t\n)都受支持。 -如果启用[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)，空的末尾加引号的输入值将替换为相应列的默认值。 +如果启用[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)，对应列如果存在未输入的空白，且没有用双引号括起，将用默认值替换。 -`NULL`被格式化为`\N`或`NULL`或一个空的非引号字符串(详见配置[input_format_csv_unquoted_null_literal_as_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null)或[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields))。 +`NULL`被格式化为`\N`或`NULL`或一个不是引号的其他字符串(详见配置[input_format_csv_unquoted_null_literal_as_null](../operations/settings/settings.md#settings-input_format_csv_unquoted_null_literal_as_null)或[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields))。 `CSV`格式支持输出总数和极值的方式与`TabSeparated`相同。 @@ -384,8 +392,8 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR ## CustomSeparated {#format-customseparated} -类似于[Template](#format-template)， 但它打印或读取所有列和使用转义规则在设置`format_custom_escaping_rule`和分隔符设置`format_custom_field_delimiter`,`format_custom_row_before_delimiter`,`format_custom_row_after_delimiter`,`format_custom_row_between_delimiter`,`format_custom_result_before_delimiter`,`format_custom_result_after_delimiter`中,而不是从格式字符串。 -也有`CustomSeparatedIgnoreSpaces`格式，这是类似于`TemplateIgnoreSpaces`。 +类似于[Template](#format-template)， 但它打印或读取所有列，并使用设置`format_custom_escaping_rule`和分隔符设置`format_custom_field_delimiter`,`format_custom_row_before_delimiter`,`format_custom_row_after_delimiter`,`format_custom_row_between_delimiter`,`format_custom_result_before_delimiter`,`format_custom_result_after_delimiter`的转义规则,而不是从格式字符串。 +也有`CustomSeparatedIgnoreSpaces`格式，这个类似于`TemplateIgnoreSpaces`。 ## JSON {#json} @@ -438,18 +446,17 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA } ``` -JSON与JavaScript兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 替代的换行符`U+2028`和`U+2029`会打断一些浏览器解析，它们会被转义为`\uXXXX`。 ASCII控制字符被转义：退格，换页，换行，回车和水平制表符被替换为`\b`，`\f`，`\n`，`\r`，`\t` 作为使用`\uXXXX`序列的00-1F范围内的剩余字节。 无效的UTF-8序列更改为替换字符，因此输出文本将包含有效的UTF-8序列。 为了与JavaScript兼容，默认情况下，Int64和UInt64整数用双引号引起来。要除去引号，可以将配置参数`output_format_json_quote_64bit_integers`设置为0。 +JSON与JavaScript兼容。为了确保这一点，一些字符被另外转义：斜线`/`被转义为`\/`; 换行符`U+2028`和`U+2029`会打断一些浏览器的解析，它们会被转义为`\uXXXX`。 ASCII控制字符被转义：退格，换页，换行，回车和制表符被转义为`\b`，`\f`，`\n`，`\r`，`\t`。剩下的0x00-0x1F被转义成相应的`\uXXXX`序列。 无效的UTF-8序列替换为字符�，使得输出文本包含有效的UTF-8序列。 为了与JavaScript兼容，默认情况下，Int64和UInt64整数用双引号引起来。要除去引号，可以将配置参数`output_format_json_quote_64bit_integers`设置为0。 -`rows` – 结果输出的行数。 +`rows`代表结果输出的行数。 -`rows_before_limit_at_least`去掉 LIMIT过滤后的最小行总数。 只会在查询包含LIMIT条件时输出。 -若查询包含 GROUP BY，`rows_before_limit_at_least`就是去掉LIMIT后过滤后的准确行数。 +`rows_before_limit_at_least`代表去掉LIMIT过滤后的最小行总数。只会在查询包含LIMIT条件时输出。若查询包含 GROUP BY，`rows_before_limit_at_least`就是去掉LIMIT后过滤后的准确行数。 -`totals` – 总值 （当使用TOTALS条件时）。 +`totals` – 总值 (当指定`WITH TOTALS`时)。 -`extremes` – 极值（当extremes设置为1时）。 +`extremes` – 极值(当extremes设置为1时)。 -该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 +该格式仅适用于输出查询结果，但不适用于解析输入(将数据插入到表中)。 ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`null`。若要在输出中启用`+nan`、`-nan`、`+inf`、`-inf`值，请设置[output_format_json_quote_denormals](../operations/settings/settings.md#settings-output_format_json_quote_denormals)为1。 @@ -506,12 +513,13 @@ ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`nu "rows_before_limit_at_least": 3 } ``` +注意range(5)的值。 ## JSONAsString {#jsonasstring} 在这种格式中，一个JSON对象被解释为一个值。如果输入有几个JSON对象(逗号分隔)，它们将被解释为独立的行。 -这种格式只能对具有单个字段类型的表进行解析[String](../sql-reference/data-types/string.md)。其余的列必须设置为[DEFAULT](../sql-reference/statements/create.md)或[MATERIALIZED](../sql-reference/statements/create.md)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。 +这种格式只能针对有一列类型为[String](../sql-reference/data-types/string.md)的表。表中其余的列必须设置为[DEFAULT](../sql-reference/statements/create.md)或[MATERIALIZED](../sql-reference/statements/create.md)，或者忽略。一旦将整个JSON对象收集为字符串，就可以使用[JSON函数](../sql-reference/functions/json-functions.md)运行它。 **示例** @@ -610,7 +618,7 @@ SELECT * FROM json_as_string; ## JSONCompactEachRow {#jsoncompacteachrow} ## JSONCompactStringEachRow {#jsoncompactstringeachrow} -使用这些格式时，ClickHouse会将行输出为分隔的、换行分隔的JSON值，但数据作为一个整体不是有效的JSON。 +使用这些格式时，ClickHouse会将行输出为用换行符分隔的JSON值，这些输出数据作为一个整体时，由于没有分隔符(,)因而不是有效的JSON文档。 ``` json {"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow @@ -635,7 +643,7 @@ SELECT * FROM json_as_string; ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} ## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} -与`JSONCompactEachRow`/`JSONCompactStringEachRow`不同的是，其中列名和类型被写入前两行。 +与`JSONCompactEachRow`/`JSONCompactStringEachRow`不同的是，列名和类型被写入前两行。 ```json ["'hello'", "multiply(42, number)", "range(5)"] @@ -653,18 +661,18 @@ INSERT INTO UserActivity FORMAT JSONEachRow {"PageViews":5, "UserID":"4324182021 ClickHouse允许: -- 对象中key-value的任何顺序。 +- 以任意顺序排列列名，后跟对应的值。 - 省略一些值。 -ClickHouse忽略元素之间的空格和对象后面的逗号。您可以在一行中传递所有对象。你不需要用换行符把它们分开。 +ClickHouse忽略元素之间的空格和对象后面的逗号。您可以在一行中传递所有对象，不需要用换行符把它们分开。 **省略值处理** -ClickHouse将省略的值替换为对应的[data types](../sql-reference/data-types/index.md)默认值。 +ClickHouse将省略的值替换为对应的[数据类型](../sql-reference/data-types/index.md)默认值。 如果指定了`DEFAULT expr`，则ClickHouse根据属性使用不同的替换规则，详看[input_format_defaults_for_omitted_fields](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields)设置。 -参考下表： +参考下面的例子： ``` sql CREATE TABLE IF NOT EXISTS example_table @@ -678,7 +686,7 @@ CREATE TABLE IF NOT EXISTS example_table - 如果`input_format_defaults_for_omitted_fields = 1`, 那么`x`的默认值为`0`，但`a`的默认值为`x * 2`。 !!! note "注意" -当使用`insert_sample_with_metadata = 1`插入数据时，与使用`insert_sample_with_metadata = 0`插入数据相比，ClickHouse消耗更多的计算资源。 +当使用`insert_sample_with_metadata = 1`插入数据时，与使用`insert_sample_with_metadata = 0`相比，ClickHouse消耗更多的计算资源。 ### Selecting Data {#selecting-data} @@ -713,13 +721,13 @@ CREATE TABLE IF NOT EXISTS example_table CREATE TABLE json_each_row_nested (n Nested (s String, i Int32) ) ENGINE = Memory ``` -正如您在`Nested`数据类型描述中看到的，ClickHouse将嵌套结构的每个组件作为一个单独的列(`n.s`和`n.i`是我们的表)。您可以通过以下方式插入数据: +正如您在`Nested`数据类型描述中看到的，ClickHouse将嵌套结构的每个部分作为一个单独的列(`n.s`和`n.i`)。您可以通过以下方式插入数据: ``` sql INSERT INTO json_each_row_nested FORMAT JSONEachRow {"n.s": ["abc", "def"], "n.i": [1, 23]} ``` -将数据作为分层JSON对象集插入[input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json)。 +将数据作为分层JSON对象集插入，需要设置[input_format_import_nested_json=1](../operations/settings/settings.md#settings-input_format_import_nested_json)。 ``` json { @@ -764,7 +772,7 @@ SELECT * FROM json_each_row_nested ## Native {#native} -最高性能的格式。通过二进制格式的块进行写入和读取。对于每个块，该中的行数，列数，列名称和类型以及列的部分将被相继记录。 换句话说，这种格式是`columnar`的 - 它不会将列转换为行。这是用于在服务器之间进行交互的本地界面中使用的格式，用于使用命令行客户端和C++客户端。 +最高性能的格式。通过二进制格式的块进行写入和读取。对于每个块，该中的行数，列数，列名称和类型以及列的部分将被相继记录。换句话说，这种格式是`columnar`的 - 它不会将列转换为行。这种格式用于服务器间交互、命令行客户端和C++客户端与服务器交互。 您可以使用此格式快速生成只能由ClickHouse DBMS读取的格式。但自己处理这种格式是没有意义的。 @@ -777,23 +785,27 @@ SELECT * FROM json_each_row_nested 将数据以表格形式输出，也可以使用ANSI转义字符在终端中设置颜色。 它会绘制一个完整的表格，每行数据在终端中占用两行。 -每个结果块作为一个单独的表输出。这是必要的，以便在输出块时不需要缓冲结果(为了预先计算所有值的可见宽度，缓冲是必要的)。 +每个结果块作为一个单独的表输出。这是必要的，以便在输出块时不缓冲结果(为了预先计算所有值的可见宽度，缓冲是必要的)。 [NULL](../sql-reference/syntax.md)输出为`ᴺᵁᴸᴸ`。 -示例(显示[PrettyCompact](#prettycompact)格式) +示例 ``` sql -SELECT * FROM t_null +SELECT * FROM system.numbers limit 2 format Pretty; ``` ``` text -┌─x─┬────y─┐ -│ 1 │ ᴺᵁᴸᴸ │ -└───┴──────┘ +┏━━━━━━━━┓ +┃ number ┃ +┡━━━━━━━━┩ +│ 0 │ +├────────┤ +│ 1 │ +└────────┘ ``` -行没有转义为Pretty\* 格式。示例显示了[PrettyCompact](#prettycompact)格式: +Pretty的所有格式不进行字符转义。示例显示了[PrettyCompact](#prettycompact)格式: ``` sql SELECT 'String with \'quotes\' and \t character' AS Escaping_test @@ -806,7 +818,7 @@ SELECT 'String with \'quotes\' and \t character' AS Escaping_test ``` 为避免将太多数据传输到终端，只打印前10,000行。 如果行数大于或等于10,000，则会显示消息`Showed first 10 000`。 -该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 +该格式仅适用于输出查询结果，但不适用于解析输入(将数据插入到表中)。 Pretty格式支持输出合计值(当使用WITH TOTALS时)和极值(当`extremes`设置为1时)。在这些情况下，合计值和极值将输出在主要数据之后，在单独的表中。示例(显示为[PrettyCompact](#prettycompact)格式): @@ -841,10 +853,20 @@ Extremes: 与[Pretty](#pretty)格式不一样的是`PrettyCompact`去掉了行之间的表格分割线，这样使得结果更加紧凑。 这种格式会在交互命令行客户端下默认使用。 +``` sql +select * from system.numbers limit 2 format PrettyCompact; +``` +``` text +┌─number─┐ +│ 0 │ +│ 1 │ +└────────┘ + +``` ## PrettyCompactMonoBlock {#prettycompactmonoblock} -与[PrettyCompact](#prettycompact)格式不一样的是，它支持10,000行数据缓冲，然后输出在一个表格中，不会按照块来区分。 +与[PrettyCompact](#prettycompact)格式不一样的是，它支持10,000行数据缓冲，然后输出在一个表格中，而不分块。 ## PrettyNoEscapes {#prettynoescapes} @@ -866,17 +888,18 @@ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FOR 用法类似上述。 -### PrettyCompactNoEscapes {#prettycompactnoescapes} - -与前面的设置相同。 - -### PrettySpaceNoEscapes {#prettyspacenoescapes} - -与前面的设置相同。 - ## PrettySpace {#prettyspace} 与[PrettyCompact](#prettycompact)格式不一样的是，它使用空格来代替网格来显示数据。 +``` sql +select * from system.numbers limit 2 format PrettySpace; +``` +``` text +number + + 0 + 1 +``` ## RowBinary {#rowbinary} @@ -885,35 +908,35 @@ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events FOR 整数使用固定长度的小端表示法。 例如，UInt64 使用8个字节。 DateTime 被表示为 UInt32 类型的Unix 时间戳值。 -Date 被表示为 UInt16 对象，它的值为 1970-01-01以来的天数。 -字符串表示为 varint 长度（无符号 [LEB128](https://en.wikipedia.org/wiki/LEB128)），后跟字符串的字节数。 -FixedString 被简单地表示为一个字节序列。 +Date 被表示为 UInt16 对象，它的值为自1970-01-01以来经过的天数。 +字符串表示为 varint 长度(无符号 [LEB128](https://en.wikipedia.org/wiki/LEB128))，后跟字符串的字节数。 +FixedString 被简单地表示为字节序列。 -数组表示为 varint 长度（无符号 [LEB128](https://en.wikipedia.org/wiki/LEB128)），后跟有序的数组元素。 +数组表示为 varint 长度(无符号 [LEB128](https://en.wikipedia.org/wiki/LEB128))，后跟有序的数组元素。 对于 [NULL](../sql-reference/syntax.md#null-literal) 的支持， 一个为 1 或 0 的字节会加在每个 [可为空](../sql-reference/data-types/nullable.md) 值前面。如果为 1, 那么该值就是 `NULL`。 如果为 0，则不为 `NULL`。 ## RowBinaryWithNamesAndTypes {#rowbinarywithnamesandtypes} -类似于 [RowBinary](#rowbinary)，但添加了标题: +类似于 [RowBinary](#rowbinary)，但添加了头部信息: -- [LEB128](https://en.wikipedia.org/wiki/LEB128)-编码列数（N) +- [LEB128](https://en.wikipedia.org/wiki/LEB128)-编码列数(N) - N `String`s指定列名 - N `String`s指定列类型 -## 值 {#data-format-values} +## Values {#data-format-values} -在括号中打印每一行。行由逗号分隔。最后一行之后没有逗号。括号内的值也用逗号分隔。数字以十进制格式输出，不含引号。 数组以方括号输出。带有时间的字符串，日期和时间用引号包围输出。转义字符的解析规则与 [TabSeparated](#tabseparated) 格式类似。 在格式化过程中，不插入额外的空格，但在解析过程中，空格是被允许并跳过的（除了数组值之外的空格，这是不允许的）。[NULL](../sql-reference/syntax.md) 为 `NULL`。 +在括号中打印每一行。行由逗号分隔。最后一行之后没有逗号。括号内的值也用逗号分隔。数字以十进制格式输出，不含引号。 数组以方括号输出。字符串、日期、日期时间用引号包围输出。转义字符的解析规则与 [TabSeparated](#tabseparated) 格式类似。 在格式化过程中，不插入额外的空格，但在解析过程中，空格是被允许并跳过的(除了数组值之外的空格，这是不允许的)。[NULL](../sql-reference/syntax.md) 为 `NULL`。 以 Values 格式传递数据时需要转义的最小字符集是：单引号和反斜线。 这是 `INSERT INTO t VALUES ...` 中可以使用的格式，但您也可以将其用于查询结果。 -## 垂直 {#vertical} +另见：[input_format_values_interpret_expressions](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-input_format_values_interpret_expressions)和[input_format_values_deduce_templates_of_expressions](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-input_format_values_deduce_templates_of_expressions)。 -使用指定的列名在单独的行上打印每个值。如果每行都包含大量列，则此格式便于打印一行或几行。 +## Vertical {#vertical} -[NULL](../sql-reference/syntax.md) 输出为 `ᴺᵁᴸᴸ`。 +根据指定的列名，打印出每一行的值。这种格式适用于具有大量的列时，显示几个列。[NULL](../sql-reference/syntax.md) 输出为 `ᴺᵁᴸᴸ`。 示例: @@ -926,12 +949,12 @@ SELECT * FROM t_null FORMAT Vertical x: 1 y: ᴺᵁᴸᴸ -该格式仅适用于输出查询结果，但不适用于解析输入（将数据插入到表中）。 +该格式仅适用于输出查询结果，但不适用于解析输入(将数据插入到表中)。 ## VerticalRaw {#verticalraw} 和 `Vertical` 格式不同点在于，行是不会被转义的。 -这种格式仅仅适用于输出，但不适用于解析输入（将数据插入到表中）。 +这种格式仅仅适用于输出，但不适用于解析输入(将数据插入到表中)。 示例: @@ -1020,24 +1043,24 @@ SELECT * FROM t_null FORMAT Vertical ``` 如果列名称没有可接受的格式，则仅使用 `field` 作为元素名称。 通常，XML 结构遵循 JSON 结构。 -就像JSON一样，将无效的 UTF-8 字符都作替换，以便输出文本将包含有效的 UTF-8 字符序列。 +就像JSON一样，将无效的 UTF-8 字符都替换成字符�，以便输出文本将包含有效的 UTF-8 字符序列。 -在字符串值中，字符 `<` 和 `＆` 被转义为 `<` 和 `＆`。 +在字符串值中，字符 `<` 和 `＆` 被转义为 `<` 和 `&`。 -数组输出为 ` Hello World ... `，元组输出为 ` Hello World ... ` 。 +数组输出类似于 ` Hello World ... `，元组输出类似于 ` Hello World ... ` 。 ## CapnProto {#capnproto} -Cap’n Proto 是一种二进制消息格式，类似 Protocol Buffers 和 Thriftis，但与 JSON 或 MessagePack 格式不一样。 +Cap’n Proto 是一种二进制消息格式，类似Protobuf和Thriftis，但与 JSON 或 MessagePack 格式不一样。 -Cap’n Proto 消息格式是严格类型的，而不是自我描述，这意味着它们不需要外部的描述。这种格式可以实时地应用，并针对每个查询进行缓存。 +Cap’n Proto 消息格式是严格类型的，而不是自我描述，这意味着它们需要架构描述。架构描述可以实时地应用，并针对每个查询进行缓存。 ``` sql SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' ``` -其中 `schema.capnp` 描述如下： +其中 `schema.capnp` 描述如下：6y2 struct Message { SearchPhrase @0 :Text; @@ -1050,10 +1073,10 @@ Cap’n Proto 反序列化是很高效的，通常不会增加系统的负载。 ## Protobuf {#protobuf} -Protobuf-是一个 [协议缓冲区](https://developers.google.com/protocol-buffers/) 格式。 +Protobuf-是一个 [Protocol Buffers](https://developers.google.com/protocol-buffers/) 格式。 -此格式需要外部格式架构。 在查询之间缓存架构。 -ClickHouse支持 `proto2` 和 `proto3` 语法 支持重复/可选/必填字段。 +此格式需要外部格式描述文件(proto文件)。 该描述文件会进行缓存，以备后续查询。 +ClickHouse支持 `proto2` 和 `proto3` 语法的proto文件，支持重复/可选/必填字段。 使用示例: @@ -1065,7 +1088,7 @@ SELECT * FROM test.table FORMAT Protobuf SETTINGS format_schema = 'schemafile:Me cat protobuf_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT Protobuf SETTINGS format_schema='schemafile:MessageType'" ``` -哪里的文件 `schemafile.proto` 看起来像这样: +proto文件 `schemafile.proto` 看起来像这样: ``` capnp syntax = "proto3"; @@ -1077,12 +1100,9 @@ message MessageType { repeated string phoneNumbers = 4; }; ``` +Clickhouse通过字段名称来对应列名称。字段名称不区分大小写，`_`与`.`视为相同符号。如果Proto文件指定的字段类型与列类型不相符，会进行转换。 -要查找协议缓冲区的消息类型的表列和字段之间的对应关系，ClickHouse比较它们的名称。 -这种比较是不区分大小写和字符 `_` (下划线)和 `.` （点）被认为是相等的。 -如果协议缓冲区消息的列和字段的类型不同，则应用必要的转换。 - -支持嵌套消息。 例如，对于字段 `z` 在下面的消息类型 +支持Protobuf嵌套消息。 例如，对于下面Proto文件中的z字段： ``` capnp message MessageType { @@ -1096,10 +1116,10 @@ message MessageType { }; ``` -ClickHouse尝试找到一个名为 `x.y.z` （或 `x_y_z` 或 `X.y_Z` 等）。 +ClickHouse会试图找到一个名为 `x.y.z` (或 `x_y_z` 或 `X.y_Z` 等)的列。 嵌套消息适用于输入或输出一个 [嵌套数据结构](../sql-reference/data-types/nested-data-structures/nested.md). -在protobuf模式中定义的默认值，如下所示 +在protobuf模式中定义的默认值，如下： ``` capnp syntax = "proto2"; @@ -1109,9 +1129,9 @@ message MessageType { } ``` -不应用;该 [表默认值](../sql-reference/statements/create.md#create-default-values) 用来代替它们。 +该默认值会被忽略，Clickhouse会使用 [表默认值](../sql-reference/statements/create.md#create-default-values)作为默认值。 -ClickHouse在输入和输出protobuf消息 `length-delimited` 格式。 +ClickHouse在输入和输出protobuf消息采用`length-delimited` 格式。 这意味着每个消息之前，应该写它的长度作为一个 [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). 另请参阅 [如何在流行语言中读取/写入长度分隔的protobuf消息](https://cwiki.apache.org/confluence/display/GEODE/Delimiting+Protobuf+Messages). @@ -1131,21 +1151,21 @@ ClickHouse Avro格式支持读取和写入 [Avro数据文件](http://avro.apache | `boolean`, `int`, `long`, `float`, `double` | [Int64](../sql-reference/data-types/int-uint.md), [UInt64](../sql-reference/data-types/int-uint.md) | `long` | | `boolean`, `int`, `long`, `float`, `double` | [Float32](../sql-reference/data-types/float.md) | `float` | | `boolean`, `int`, `long`, `float`, `double` | [Float64](../sql-reference/data-types/float.md) | `double` | -| `bytes`, `string`, `fixed`, `enum` | [字符串](../sql-reference/data-types/string.md) | `bytes` | -| `bytes`, `string`, `fixed` | [固定字符串(N)](../sql-reference/data-types/fixedstring.md) | `fixed(N)` | -| `enum` | [枚举(8/16)](../sql-reference/data-types/enum.md) | `enum` | -| `array(T)` | [阵列(T)](../sql-reference/data-types/array.md) | `array(T)` | -| `union(null, T)`, `union(T, null)` | [可为空(T)](../sql-reference/data-types/date.md) | `union(null, T)` | -| `null` | [可为空（无)](../sql-reference/data-types/special-data-types/nothing.md) | `null` | -| `int (date)` \* | [日期](../sql-reference/data-types/date.md) | `int (date)` \* | +| `bytes`, `string`, `fixed`, `enum` | [String](../sql-reference/data-types/string.md) | `bytes` | +| `bytes`, `string`, `fixed` | [FixedString(N)](../sql-reference/data-types/fixedstring.md) | `fixed(N)` | +| `enum` | [Enum(8\|16)](../sql-reference/data-types/enum.md) | `enum` | +| `array(T)` | [Array(T)](../sql-reference/data-types/array.md) | `array(T)` | +| `union(null, T)`, `union(T, null)` | [Nullable(T)](../sql-reference/data-types/date.md) | `union(null, T)` | +| `null` | [Nullable(Nothing)](../sql-reference/data-types/special-data-types/nothing.md) | `null` | +| `int (date)` \* | [Date](../sql-reference/data-types/date.md) | `int (date)` \* | | `long (timestamp-millis)` \* | [DateTime64(3)](../sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | | `long (timestamp-micros)` \* | [DateTime64(6)](../sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | \* [Avro逻辑类型](http://avro.apache.org/docs/current/spec.html#Logical+Types) -不支持的Avro数据类型: `record` （非根), `map` +不支持的Avro数据类型: `record` (非根架构), `map` -不支持的Avro逻辑数据类型: `uuid`, `time-millis`, `time-micros`, `duration` +不支持的Avro逻辑数据类型: `time-millis`, `time-micros`, `duration` ### 插入数据 {#inserting-data} @@ -1155,12 +1175,11 @@ ClickHouse Avro格式支持读取和写入 [Avro数据文件](http://avro.apache $ cat file.avro | clickhouse-client --query="INSERT INTO {some_table} FORMAT Avro" ``` -输入Avro文件的根模式必须是 `record` 类型。 +输入Avro文件的根架构必须是 `record` 类型。 -要查找Avro schema的表列和字段之间的对应关系，ClickHouse比较它们的名称。 此比较区分大小写。 -跳过未使用的字段。 +Clickhouse通过字段名称来对应架构的列名称。字段名称区分大小写。未使用的字段会被跳过。 -ClickHouse表列的数据类型可能与插入的Avro数据的相应字段不同。 插入数据时，ClickHouse根据上表解释数据类型，然后 [投](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 将数据转换为相应的列类型。 +ClickHouse表列的数据类型可能与插入的Avro数据的相应字段不同。 插入数据时，ClickHouse根据上表解释数据类型，然后通过 [Cast](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 将数据转换为相应的列类型。 ### 选择数据 {#selecting-data} @@ -1172,14 +1191,14 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Avro" > file.avro 列名必须: -- 名,名,名,名 `[A-Za-z_]` +- 以 `[A-Za-z_]` 开始 - 随后只包含 `[A-Za-z0-9_]` -输出Avro文件压缩和同步间隔可以配置 [output_format_avro_codec](../operations/settings/settings.md#settings-output_format_avro_codec) 和 [output_format_avro_sync_interval](../operations/settings/settings.md#settings-output_format_avro_sync_interval) 分别。 +输出Avro文件压缩和同步间隔可以经由 [output_format_avro_codec](../operations/settings/settings.md#settings-output_format_avro_codec) 和 [output_format_avro_sync_interval](../operations/settings/settings.md#settings-output_format_avro_sync_interval) 设置。 ## AvroConfluent {#data-format-avro-confluent} -AvroConfluent支持解码单对象Avro消息常用于 [卡夫卡](https://kafka.apache.org/) 和 [汇合的模式注册表](https://docs.confluent.io/current/schema-registry/index.html). +AvroConfluent支持解码单个对象的Avro消息，这常用于 [Kafka](https://kafka.apache.org/) 和 [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/index.html)。 每个Avro消息都嵌入了一个架构id，该架构id可以在架构注册表的帮助下解析为实际架构。 @@ -1189,11 +1208,11 @@ AvroConfluent支持解码单对象Avro消息常用于 [卡夫卡](https://kafka. ### 数据类型匹配{#sql_reference/data_types-matching-1} {#data-types-matching-sql_referencedata_types-matching-1} -和 [Avro](#data-format-avro) +和 [Avro](#data-format-avro)相同。 ### 用途 {#usage} -要快速验证架构解析，您可以使用 [kafkacat](https://github.com/edenhill/kafkacat) 与 [ﾂ环板-ｮﾂ嘉ｯﾂ偲](../operations/utilities/clickhouse-local.md): +要快速验证架构解析，您可以使用 [kafkacat](https://github.com/edenhill/kafkacat) 与 [clickhouse-local](../operations/utilities/clickhouse-local.md): ``` bash $ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse-local --input-format AvroConfluent --format_avro_schema_registry_url 'http://schema-registry' -S "field1 Int64, field2 String" -q 'select * from table' @@ -1202,7 +1221,7 @@ $ kafkacat -b kafka-broker -C -t topic1 -o beginning -f '%s' -c 3 | clickhouse- 3 c ``` -使用 `AvroConfluent` 与 [卡夫卡](../engines/table-engines/integrations/kafka.md): +使用 `AvroConfluent` 与 [Kafka](../engines/table-engines/integrations/kafka.md): ``` sql CREATE TABLE topic1_stream @@ -1223,15 +1242,16 @@ SELECT * FROM topic1_stream; ``` !!! note "警告" - 设置 `format_avro_schema_registry_url` 需要在配置 `users.xml` restart动后保持它的价值。 + 设置 `format_avro_schema_registry_url` 需要写入配置文件`users.xml`以在Clickhouse重启后，该设置仍为您的设定值。您也可以在使用Kafka引擎的时候指定该设置。 + ## Parquet {#data-format-parquet} -[Apache Parquet](http://parquet.apache.org/) 是Hadoop生态系统中普遍存在的列式存储格式。 ClickHouse支持此格式的读写操作。 +[Apache Parquet](http://parquet.apache.org/) 是Hadoop生态系统中普遍使用的列式存储格式。 ClickHouse支持此格式的读写操作。 ### 数据类型匹配{#sql_reference/data_types-matching-2} {#data-types-matching-sql_referencedata_types-matching-2} -下表显示了支持的数据类型以及它们如何匹配ClickHouse [数据类型](../sql-reference/data-types/index.md) 在 `INSERT` 和 `SELECT` 查询。 +下表显示了Clickhouse支持的数据类型以及它们在 `INSERT` 和 `SELECT` 查询如何对应Clickhouse的 [data types](../sql-reference/data-types/index.md) 。 | Parquet数据类型 (`INSERT`) | ClickHouse数据类型 | Parquet数据类型 (`SELECT`) | |----------------------------|----------------------------------------------------------|----------------------------| @@ -1245,17 +1265,17 @@ SELECT * FROM topic1_stream; | `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | `INT64` | | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | -| `DATE32` | [日期](../sql-reference/data-types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [日期时间](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [字符串](../sql-reference/data-types/string.md) | `STRING` | -| — | [固定字符串](../sql-reference/data-types/fixedstring.md) | `STRING` | -| `DECIMAL` | [十进制](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | +| — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | +| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | -ClickHouse支持可配置的精度 `Decimal` 类型。 该 `INSERT` 查询对待 Parquet `DECIMAL` 键入为ClickHouse `Decimal128` 类型。 +ClickHouse支持对 `Decimal` 类型设置精度。 `INSERT` 查询将 Parquet `DECIMAL` 类型视为ClickHouse `Decimal128` 类型。 不支持的Parquet数据类型: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -ClickHouse表列的数据类型可能与插入的Parquet数据的相应字段不同。 插入数据时，ClickHouse根据上表解释数据类型，然后 [投](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 为ClickHouse表列设置的数据类型的数据。 +ClickHouse表列的数据类型可能与插入的Parquet数据的相应字段不同。 插入数据时，ClickHouse根据上表解释数据类型，然后 [Cast](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 为ClickHouse表列设置的数据类型的数据。 ### 插入和选择数据 {#inserting-and-selecting-data} @@ -1265,44 +1285,52 @@ ClickHouse表列的数据类型可能与插入的Parquet数据的相应字段不 $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` -您可以从ClickHouse表中选择数据，并通过以下命令将它们保存到Parquet格式的某个文件中: +您可以从ClickHouse表中选择数据，并通过以下命令将它们保存到Parquet格式的文件中: ``` bash $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_file.pq} ``` -要与Hadoop交换数据，您可以使用 [HDFS表引擎](../engines/table-engines/integrations/hdfs.md). +要与Hadoop交换数据，您可以使用 [HDFS table engine](../engines/table-engines/integrations/hdfs.md). + +## Arrow {#data-format-arrow} +[Apache Arrow](https://arrow.apache.org/)是一种用于内存数据库的格式，共有两种模式，文件与流模式。Clickhouse支持对这两种格式进行读写。 + +`Arrow`对应的是文件模式，这种格式适用于内存的随机访问。 + +## ArrowStream {#data-format-arrow} +`ArrowStream`对应的是Arrow的流模式，这种格式适用于内存的流式处理。 ## ORC {#data-format-orc} - -[阿帕奇兽人](https://orc.apache.org/) 是Hadoop生态系统中普遍存在的列式存储格式。 您只能将此格式的数据插入ClickHouse。 +[Apache ORC](https://orc.apache.org/) 是Hadoop生态系统中普遍存在的列式存储格式。 ### 数据类型匹配{#sql_reference/data_types-matching-3} {#data-types-matching-sql_referencedata_types-matching-3} -下表显示了支持的数据类型以及它们如何匹配ClickHouse [数据类型](../sql-reference/data-types/index.md) 在 `INSERT` 查询。 +下表显示了支持的数据类型以及它们如何在`SELECT`与`INSERT`查询中匹配ClickHouse的 [数据类型](../sql-reference/data-types/index.md)。 -| ORC数据类型 (`INSERT`) | ClickHouse数据类型 | -|------------------------|-----------------------------------------------------| -| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | -| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | -| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | -| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | -| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | -| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | -| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | -| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | -| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | -| `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | -| `DATE32` | [日期](../sql-reference/data-types/date.md) | -| `DATE64`, `TIMESTAMP` | [日期时间](../sql-reference/data-types/datetime.md) | -| `STRING`, `BINARY` | [字符串](../sql-reference/data-types/string.md) | -| `DECIMAL` | [十进制](../sql-reference/data-types/decimal.md) | +| ORC 数据类型 (`INSERT`) | ClickHouse 数据类型 | ORC 数据类型 (`SELECT`) | +|--------------------------|-----------------------------------------------------|--------------------------| +| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | `UINT8` | +| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | `INT8` | +| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | `UINT16` | +| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | `INT16` | +| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | `UINT32` | +| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | `INT32` | +| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | `UINT64` | +| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | `INT64` | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | +| `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | +| `DATE32` | [Date](../sql-reference/data-types/date.md) | `DATE32` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `TIMESTAMP` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | +| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `-` | [Array](../sql-reference/data-types/array.md) | `LIST` | -ClickHouse支持的可配置精度 `Decimal` 类型。 该 `INSERT` 查询对待兽人 `DECIMAL` 键入为ClickHouse `Decimal128` 类型。 +ClickHouse支持的可配置精度的 `Decimal` 类型。 `INSERT` 查询将ORC格式的 `DECIMAL` 类型视为ClickHouse的 `Decimal128` 类型。 -不支持的ORC数据类型: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +不支持的ORC数据类型: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -ClickHouse表列的数据类型不必匹配相应的ORC数据字段。 插入数据时，ClickHouse根据上表解释数据类型，然后 [投](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 将数据转换为ClickHouse表列的数据类型集。 +ClickHouse表列的数据类型不必匹配相应的ORC数据字段。 插入数据时，ClickHouse根据上表解释数据类型，然后 [Cast](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) 将数据转换为ClickHouse表列的数据类型集。 ### 插入数据 {#inserting-data-1} @@ -1312,33 +1340,143 @@ ClickHouse表列的数据类型不必匹配相应的ORC数据字段。 插入数 $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` +### 选择数据 {#selecting-data-1} + +您可以通过以下命令将ClickHouse表中某些数据导出到ORC文件: + +``` bash +$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT ORC" > {filename.orc} +``` + 要与Hadoop交换数据，您可以使用 [HDFS表引擎](../engines/table-engines/integrations/hdfs.md). +## LineAsString {#lineasstring} +这种格式下，每行输入数据都会当做一个字符串。这种格式仅适用于仅有一列[String](https://clickhouse.tech/docs/en/sql-reference/data-types/string/)类型的列的表。其余列必须设置为[DEFAULT](https://clickhouse.tech/docs/en/sql-reference/statements/create/table/#default)、[MATERIALIZED](https://clickhouse.tech/docs/en/sql-reference/statements/create/table/#materialized)或者被忽略。 + +### 示例： +查询如下： +``` sql +DROP TABLE IF EXISTS line_as_string; +CREATE TABLE line_as_string (field String) ENGINE = Memory; +INSERT INTO line_as_string FORMAT LineAsString "I love apple", "I love banana", "I love orange"; +SELECT * FROM line_as_string; +``` +结果如下： +``` text +┌─field─────────────────────────────────────────────┐ +│ "I love apple", "I love banana", "I love orange"; │ +└───────────────────────────────────────────────────┘ +``` +## Regexp {#regexp} +每一列输入数据根据正则表达式解析。使用`Regexp`格式时，可以使用如下设置： + +- `format_regexp`，[String](https://clickhouse.tech/docs/en/sql-reference/data-types/string/)类型。包含[re2](https://github.com/google/re2/wiki/Syntax)格式的正则表达式。 +- `format_regexp_escaping_rule`，[String](https://clickhouse.tech/docs/en/sql-reference/data-types/string/)类型。支持如下转义规则： + - CSV(规则相同于[CSV](https://clickhouse.tech/docs/zh/interfaces/formats/#csv)) + - JSON(相同于[JSONEachRow](https://clickhouse.tech/docs/zh/interfaces/formats/#jsoneachrow)) + - Escaped(相同于[TSV](https://clickhouse.tech/docs/zh/interfaces/formats/#tabseparated)) + - Quoted(相同于[Values](https://clickhouse.tech/docs/zh/interfaces/formats/#data-format-values)) + - Raw(将整个子匹配项进行提取，不转义) +- `format_regexp_skip_unmatched`，[UInt8](https://clickhouse.tech/docs/zh/sql-reference/data-types/int-uint/)类型。当`format_regexp`表达式没有匹配到结果时是否抛出异常。可为0或1。 + +### 用法 {#usage-1} +`format_regexp`设置会应用于每一行输入数据。正则表达式的子匹配项数必须等于输入数据期望得到的列数。 +每一行输入数据通过换行符`\n`或者`\r\n`分隔。 +匹配到的子匹配项会根据每一列的数据格式进行解析，转义规则根据`format_regexp_escaping_rule`进行。 +当正则表达式对某行没有匹配到结果，`format_regexp_skip_unmatched`设为1时，该行会被跳过。`format_regexp_skip_unmatched`设为0时，会抛出异常。 + +### 示例 {#example-1} +设有如下data.tsv: +``` text +id: 1 array: [1,2,3] string: str1 date: 2020-01-01 +id: 2 array: [1,2,3] string: str2 date: 2020-01-02 +id: 3 array: [1,2,3] string: str3 date: 2020-01-03 +``` +与表: +``` sql +CREATE TABLE imp_regex_table (id UInt32, array Array(UInt32), string String, date Date) ENGINE = Memory; +``` +导入命令： +``` bash +$ cat data.tsv | clickhouse-client --query "INSERT INTO imp_regex_table FORMAT Regexp SETTINGS format_regexp='id: (.+?) array: (.+?) string: (.+?) date: (.+?)', format_regexp_escaping_rule='Escaped', format_regexp_skip_unmatched=0;" +``` +查询： +``` sql +SELECT * FROM imp_regex_table; +``` +结果： +``` text +┌─id─┬─array───┬─string─┬───────date─┐ +│ 1 │ [1,2,3] │ str1 │ 2020-01-01 │ +│ 2 │ [1,2,3] │ str2 │ 2020-01-02 │ +│ 3 │ [1,2,3] │ str3 │ 2020-01-03 │ +└────┴─────────┴────────┴────────────┘ +``` + + + +## RawBLOB {#rawblob} +这种格式下，所有输入数据视为一个值。该格式仅适用于仅有一String类型的列的表。输出时，使用二进制格式输出。当输出结果不唯一时，输出是有歧义的，并且不能通过该输出还原原数据。 +下面是`RawBLOB`与[TabSeparatedRaw](https://clickhouse.tech/docs/zh/interfaces/formats/#tabseparatedraw)的对比： + +`RawBloB`: + +- 二进制格式输出，无转义。 +- 值之间没有分隔符。 +- 每行最后的值后面没有换行符。 + +`TabSeparatedRaw`: + +- 数据无转义输出。 +- 每行的值通过制表符分隔。 +- 每行最后的值得后面有换行符。 + +下面是`RawBLOB`与[RowBinary](https://clickhouse.tech/docs/zh/interfaces/formats/#rowbinary)的对比： + +`RawBloB`: + +- 字符串前面没有表示长度的标志 + +`RowBinary`: + +- 字符串前面有变长标志([LEB128](https://en.wikipedia.org/wiki/LEB128)格式表示)，用于表示字符串长度，后接字符串内容。 + +当传入空数据，Clickhouse会抛出异常： +``` text +Code: 108. DB::Exception: No data to insert +``` +### 示例 {#example-4} +``` bash +$ clickhouse-client --query "CREATE TABLE {some_table} (a String) ENGINE = Memory;" +$ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT RawBLOB" +$ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum +``` +结果： +``` text +f9725a22f9191e064120d718e26862a9 - +``` + ## 格式架构 {#formatschema} -包含格式架构的文件名由该设置设置 `format_schema`. -当使用其中一种格式时，需要设置此设置 `Cap'n Proto` 和 `Protobuf`. -格式架构是文件名和此文件中消息类型的名称的组合，用冒号分隔, -e.g. `schemafile.proto:MessageType`. -如果文件具有格式的标准扩展名（例如, `.proto` 为 `Protobuf`), -它可以被省略，在这种情况下，格式模式如下所示 `schemafile:MessageType`. +包含格式架构的文件名由设置 `format_schema`指定.当使用`CapnProto` 或 `Protobuf`其中一种格式时，需要设置该项. +格式架构为架构文件名和此文件中消息类型的组合，用冒号分隔,例如 `schemafile.proto:MessageType`. +如果文件具有格式的标准扩展名(例如, `Protobuf`格式的架构文件标准扩展名为`.proto`),它可以被省略，在这种情况下，格式模式如下所示 `schemafile:MessageType`. -如果您通过输入或输出数据 [客户](../interfaces/cli.md) 在交互模式下，格式架构中指定的文件名 -可以包含绝对路径或相对于客户端上当前目录的路径。 -如果在批处理模式下使用客户端，则由于安全原因，架构的路径必须是相对的。 +如果您通过[Client](../interfaces/cli.md) 在 [交互模式](https://clickhouse.tech/docs/zh/interfaces/cli/#cli_usage)下输入或输出数据，格式架构中指定的文件名可以使用绝对路径或客户端当前目录的相对路径。 +如果在[批处理模式](https://clickhouse.tech/docs/zh/interfaces/cli/#cli_usage)下使用客户端，则由于安全原因，架构的路径必须使用相对路径。 -如果您通过输入或输出数据 [HTTP接口](../interfaces/http.md) 格式架构中指定的文件名 -应该位于指定的目录中 [format_schema_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path) -在服务器配置中。 +如果您通过 HTTP接口](../interfaces/http.md)输入或输出数据，格式架构中指定的文件名应该位于服务器设置的[format_schema_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path)指定的目录中。 -[原始文章](https://clickhouse.tech/docs/en/interfaces/formats/) ## 跳过错误 {#skippingerrors} -一些格式，如 `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` 和 `Protobuf` 如果发生解析错误，可以跳过断开的行，并从下一行开始继续解析。 看 [input_format_allow_errors_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) 和 -[input_format_allow_errors_ratio](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) 设置。 +一些格式，如 `CSV`, `TabSeparated`, `TSKV`, `JSONEachRow`, `Template`, `CustomSeparated` 和 `Protobuf` 如果发生解析错误，可以跳过引发错误的行，并从下一行开始继续解析。 详情请见设置[input_format_allow_errors_num](../operations/settings/settings.md#settings-input_format_allow_errors_num) 和 +[input_format_allow_errors_ratio](../operations/settings/settings.md#settings-input_format_allow_errors_ratio) 。 + 限制: --在解析错误的情况下 `JSONEachRow` 跳过所有数据，直到新行（或EOF），所以行必须由 `\n` 正确计算错误。 -- `Template` 和 `CustomSeparated` 在最后一列之后使用分隔符，并在行之间使用分隔符来查找下一行的开头，所以跳过错误只有在其中至少有一个不为空时才有效。 +- 在解析错误的情况下 `JSONEachRow` 跳过该行的所有数据，直到遇到新行(或EOF)，所以行必须由换行符分隔以正确统计错误行的数量。 +- `Template` 和 `CustomSeparated` 在最后一列之后和行之间使用分隔符来查找下一行的开头，所以跳过错误只有在行分隔符和列分隔符其中至少有一个不为空时才有效。 + + [来源文章](https://clickhouse.tech/docs/zh/interfaces/formats/) From 68b20a934aa136a6ee85e8542b52a68e09b97a1d Mon Sep 17 00:00:00 2001 From: AnaUvarova <64017504+AnaUvarova@users.noreply.github.com> Date: Tue, 15 Jun 2021 15:21:44 +0300 Subject: [PATCH 194/491] DOCSUP-6543: Edit and translate Show Clusters and MergeTree (#24130) * + * ready for review * Update docs/ru/sql-reference/statements/show.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update merge-tree-settings.md * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update merge-tree-settings.md * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> * Update merge-tree-settings.md * about fsync * Update merge-tree-settings.md * Update merge-tree-settings.md * Update merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/clean | 0 .../settings/merge-tree-settings.md | 9 +- docs/en/sql-reference/statements/show.md | 4 +- .../settings/merge-tree-settings.md | 85 ++++++++++--------- docs/ru/sql-reference/statements/show.md | 74 +++++++++++++++- 5 files changed, 127 insertions(+), 45 deletions(-) create mode 100644 docs/clean diff --git a/docs/clean b/docs/clean new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index fc5c887c92e..791ac344bcf 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -191,10 +191,12 @@ Possible values: Default value: 480. -`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged. -To protect data parts created by merges source parts are not deleted immediately. After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds. +After merging several parts into a new part, ClickHouse marks the original parts as inactive and deletes them only after `old_parts_lifetime` seconds. Inactive parts are removed if they are not used by current queries, i.e. if the `refcount` of the part is zero. +`fsync` is not called for new parts, so for some time new parts exist only in the server's RAM (OS cache). If the server is rebooted spontaneously, new parts can be lost or damaged. +To protect data inactive parts are not deleted immediately. + During startup ClickHouse checks the integrity of the parts. If the merged part is damaged ClickHouse returns the inactive parts to the active list, and later merges them again. Then the damaged part is renamed (the `broken_` prefix is added) and moved to the `detached` folder. If the merged part is not damaged, then the original inactive parts are renamed (the `ignored_` prefix is added) and moved to the `detached` folder. @@ -214,7 +216,7 @@ Default value: 161061273600 (150 GB). The merge scheduler periodically analyzes the sizes and number of parts in partitions, and if there is enough free resources in the pool, it starts background merges. Merges occur until the total size of the source parts is less than `max_bytes_to_merge_at_max_space_in_pool`. -Merges initiated by `optimize final` ignore `max_bytes_to_merge_at_max_space_in_pool` and merge parts only taking into account available resources (free disk's space) until one part remains in the partition. +Merges initiated by [OPTIMIZE FINAL](../../sql-reference/statements/optimize.md) ignore `max_bytes_to_merge_at_max_space_in_pool` and merge parts only taking into account available resources (free disk's space) until one part remains in the partition. ## max_bytes_to_merge_at_min_space_in_pool {#max-bytes-to-merge-at-min-space-in-pool} @@ -252,6 +254,7 @@ Possible values: Default value: auto (number of CPU cores). During startup ClickHouse reads all parts of all tables (reads files with metadata of parts) to build a list of all parts in memory. In some systems with a large number of parts this process can take a long time, and this time might be shortened by increasing `max_part_loading_threads` (if this process is not CPU and disk I/O bound). + ## max_partitions_to_read {#max-partitions-to-read} Limits the maximum number of partitions that can be accessed in one query. diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index a78ef38241f..eaded449a33 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -366,9 +366,9 @@ Returns a list of clusters. All available clusters are listed in the [system.clu ``` sql SHOW CLUSTER '' -SWOW CLUSTERS [LIKE|NOT LIKE ''] [LIMIT ] +SHOW CLUSTERS [LIKE|NOT LIKE ''] [LIMIT ] ``` -### Examples +### Examples {#show-cluster-examples} Query: diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 4ef811eb1dc..9ae247cf7a7 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -1,6 +1,6 @@ # Настройки MergeTree таблиц {#merge-tree-settings} -Значения настроек для всех MergeTree таблиц можно посмотреть в таблице `system.merge_tree_settings`, их можно переопределить в `config.xml` в секции `merge_tree`, или задать в секции `SETTINGS` у каждой таблицы. +Значения настроек всех MergeTree таблиц собраны в таблице `system.merge_tree_settings`. Их можно переопределить в разделе `merge_tree` файла `config.xml` или задать в секции `SETTINGS` каждой таблицы. Пример переопределения в `config.xml`: @@ -10,7 +10,7 @@ ``` -Пример для определения в `SETTINGS` у конкретной таблицы: +Пример установки `SETTINGS` для конкретной таблицы: ``` sql CREATE TABLE foo @@ -22,7 +22,7 @@ ORDER BY tuple() SETTINGS max_suspicious_broken_parts = 500; ``` -Пример изменения настроек у конкретной таблицы командой `ALTER TABLE ... MODIFY SETTING`: +Пример изменения настроек для конкретной таблицы при помощи команды `ALTER TABLE ... MODIFY SETTING`: ``` sql ALTER TABLE foo @@ -31,7 +31,7 @@ ALTER TABLE foo ## parts_to_throw_insert {#parts-to-throw-insert} -Eсли число кусков в партиции превышает значение `parts_to_throw_insert`, INSERT прерывается с исключением `Too many parts (N). Merges are processing significantly slower than inserts`. +Eсли число активных кусков в партиции больше значения `parts_to_throw_insert`, то INSERT прерывается с исключением: `Too many parts (N). Merges are processing significantly slower than inserts`. Возможные значения: @@ -39,13 +39,13 @@ Eсли число кусков в партиции превышает знач Значение по умолчанию: 300. -Для достижения максимальной производительности запросов `SELECT` необходимо минимизировать количество обрабатываемых кусков, см. [Дизайн MergeTree](../../development/architecture.md#merge-tree). +Чтобы производительность запросов `SELECT` стала максимальной, необходимо минимизировать количество обрабатываемых кусков, см. [Дизайн MergeTree](../../development/architecture.md#merge-tree). -Можно установить большее значение 600 (1200), это уменьшит вероятность возникновения ошибки `Too many parts`, но в тоже время вы позже обнаружите возможную проблему со слияниями (например, из-за недостатка места на диске) и деградацию производительности `SELECT`. +Можно установить значение больше — 600 (1200) кусков. Тогда ошибка `Too many parts` будет появляться реже, но при этом могут возникнуть проблемы с фоновыми слияниями и производительностью `SELECT`-запросов. ## parts_to_delay_insert {#parts-to-delay-insert} -Eсли число кусков в партиции превышает значение `parts_to_delay_insert`, `INSERT` искусственно замедляется. +Eсли число кусков в партиции больше значения `parts_to_delay_insert`, то `INSERT` искусственно замедляется. Возможные значения: @@ -53,31 +53,31 @@ Eсли число кусков в партиции превышает знач Значение по умолчанию: 150. -ClickHouse искусственно выполняет `INSERT` дольше (добавляет ‘sleep’), чтобы фоновый механизм слияния успевал слиять куски быстрее, чем они добавляются. +ClickHouse искусственно выполняет `INSERT` дольше (добавляет ‘sleep’) так, чтобы куски сливались в фоновом процессе быстрее, чем добавляются. ## inactive_parts_to_throw_insert {#inactive-parts-to-throw-insert} -Если число неактивных кусков в партиции превышает значение `inactive_parts_to_throw_insert`, `INSERT` прерывается с исключением «Too many inactive parts (N). Parts cleaning are processing significantly slower than inserts». +Если число неактивных кусков в партиции больше значения `inactive_parts_to_throw_insert`, то `INSERT` прерывается с исключением `Too many inactive parts (N). Parts cleaning are processing significantly slower than inserts`. Возможные значения: - Положительное целое число. -Значение по умолчанию: 0 (не ограничено). +Значение по умолчанию: 0 (без ограничений). ## inactive_parts_to_delay_insert {#inactive-parts-to-delay-insert} -Если число неактивных кусков в партиции больше или равно значению `inactive_parts_to_delay_insert`, `INSERT` искусственно замедляется. Это полезно, когда сервер не может быстро очистить неактивные куски. +Если число неактивных кусков в партиции больше или равно значению `inactive_parts_to_delay_insert`, то `INSERT` искусственно замедляется. Это помогает, когда сервер не может быстро очистить неактивные куски. Возможные значения: - Положительное целое число. -Значение по умолчанию: 0 (не ограничено). +Значение по умолчанию: 0 (без ограничений). ## max_delay_to_insert {#max-delay-to-insert} -Величина в секундах, которая используется для расчета задержки `INSERT`, если число кусков в партиции превышает значение [parts_to_delay_insert](#parts-to-delay-insert). +Величина в секундах, которая используется для расчета задержки `INSERT` в случаях, когда число кусков в партиции больше значения [parts_to_delay_insert](#parts-to-delay-insert). Возможные значения: @@ -87,17 +87,17 @@ ClickHouse искусственно выполняет `INSERT` дольше (д Величина задержки (в миллисекундах) для `INSERT` вычисляется по формуле: -``` code +```code max_k = parts_to_throw_insert - parts_to_delay_insert k = 1 + parts_count_in_partition - parts_to_delay_insert delay_milliseconds = pow(max_delay_to_insert * 1000, k / max_k) ``` -Т.е. если в партиции уже 299 кусков и parts_to_throw_insert = 300, parts_to_delay_insert = 150, max_delay_to_insert = 1, `INSERT` замедлится на `pow( 1 * 1000, (1 + 299 - 150) / (300 - 150) ) = 1000` миллисекунд. +Т.е. если в партиции уже 299 кусков и parts_to_throw_insert = 300, parts_to_delay_insert = 150, а max_delay_to_insert = 1, то `INSERT` замедлится на `pow( 1 * 1000, (1 + 299 - 150) / (300 - 150) ) = 1000` миллисекунд. ## max_parts_in_total {#max-parts-in-total} -Eсли суммарное число активных кусков во всех партициях таблицы превышает значение `max_parts_in_total`, INSERT прерывается с исключением `Too many parts (N)`. +Eсли суммарное число активных кусков во всех партициях таблицы больше значения `max_parts_in_total`, то INSERT прерывается с исключением `Too many parts (N)`. Возможные значения: @@ -105,20 +105,22 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 100000. -Большое число кусков в таблице снижает производительность запросов ClickHouse и увеличивает время старта ClickHouse. Чаще всего это следствие неправильного дизайна (ошибки при выборе стратегии партиционирования -- слишком мелкие партиции). +С большим числом кусков в таблице производительность запросов ClickHouse снижается, а время старта ClickHouse — увеличивается. Чаще всего это следствие неправильного дизайна (ошибки выбора стратегии партиционирования, например, слишком мелкие партиции). ## replicated_deduplication_window {#replicated-deduplication-window} -Количество хеш-сумм последних вставленных блоков, хранящихся в Zookeeper. +Количество хеш-сумм последних вставленных блоков, которые хранятся в Zookeeper. Возможные значения: - Положительное целое число. +- 0 (без ограничений). Значение по умолчанию: 100. -Команда `Insert` создает один или несколько блоков (кусков). При вставке в Replicated таблицы ClickHouse для [дедупликации вставок](../../engines/table-engines/mergetree-family/replication.md) записывает в Zookeeper хеш-суммы созданных кусков. Но хранятся хеш-суммы не всех кусков, а только последние `replicated_deduplication_window`. Наиболее старые хеш-суммы удаляются из Zookeeper. -Большое число `replicated_deduplication_window` замедляет `Insert`-ы. Хеш-сумма рассчитывается от композиции имен и типов полей, а также данных вставленного куска (потока байт). +Команда `Insert` создает один или несколько блоков (кусков). При вставке в Replicated таблицы ClickHouse для [дедупликации вставок](../../engines/table-engines/mergetree-family/replication.md) записывает в Zookeeper хеш-суммы созданных кусков. Но хранятся только последние `replicated_deduplication_window` хеш-сумм. Самые старые хеш-суммы удаляются из Zookeeper. +Большое значение `replicated_deduplication_window` замедляет `Insert`, так как приходится сравнивать большее количество хеш-сумм. +Хеш-сумма рассчитывается по названиям и типам полей, а также по данным вставленного куска (потока байт). ## non_replicated_deduplication_window {#non-replicated-deduplication-window} @@ -135,7 +137,7 @@ Eсли суммарное число активных кусков во все ## replicated_deduplication_window_seconds {#replicated-deduplication-window-seconds} -Число секунд, после которых хеш-суммы вставленных блоков удаляются из Zookeeper. +Время хранения (в секундах) хеш-сумм вставленных блоков в Zookeeper. По истечении этого времени хеш-суммы удаляются. Возможные значения: @@ -143,11 +145,11 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 604800 (1 неделя). -Аналогично [replicated_deduplication_window](#replicated-deduplication-window), задает, сколько времени хранить хеш-суммы блоков для дедупликции `Insert`-в. Хеш-суммы старше `replicated_deduplication_window_seconds` удаляются из Zookeeper, даже если их меньше чем `replicated_deduplication_window`. +Аналогично [replicated_deduplication_window](#replicated-deduplication-window), настройка `replicated_deduplication_window_seconds` задает время хранения хеш-сумм блоков для дедупликции `Insert`. Хеш-суммы старше значения `replicated_deduplication_window_seconds` удаляются из Zookeeper, даже если количество оставшихся хеш-сумм станет меньше чем `replicated_deduplication_window`. ## old_parts_lifetime {#old-parts-lifetime} -Время (в секундах) хранения неактивных кусков, для защиты от потери данных при спонтанной перезагрузке сервера или О.С. +Время (в секундах) хранения неактивных кусков для защиты от потери данных при спонтанной перезагрузке сервера. Возможные значения: @@ -155,12 +157,16 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 480. -После слияния нескольких кусков в новый кусок, ClickHouse помечает исходные куски как неактивные и удаляет их после `old_parts_lifetime` секунд. -Неактивные куски удаляются, если они не используются в текущих запросах, т.е. если счетчик ссылок куска – `refcount` равен нулю. +После объединения нескольких кусков в один новый ClickHouse помечает исходные куски как неактивные и удаляет их по прошествии `old_parts_lifetime` секунд. +Неактивные куски удаляются, если они не нужны для текущих запросов, т.е. если счетчик ссылок куска `refcount` имеет нулевое значение. -Неактивные куски удаляются не сразу, потому что при записи нового куска не вызывается `fsync`, т.е. некоторое время новый кусок находится только в оперативной памяти сервера (кеше О.С.). Т.о. при спонтанной перезагрузке сервера новый (смерженный) кусок может быть потерян или испорчен. В этом случае ClickHouse в процессе старта при проверке целостности кусков обнаружит проблему, вернет неактивные куски в список активных и позже заново их смержит. Сломанный кусок в этом случае переименовывается (добавляется префикс broken_) и перемещается в папку detached. Если проверка целостности не обнаруживает проблем в смерженном куске, то исходные неактивные куски переименовываются (добавляется префикс ignored_) и перемещаются в папку detached. +При записи нового куска `fsync` не вызывается, поэтому неактивные куски удаляются позже. Это значит, что некоторое время новый кусок находится только в оперативной памяти сервера (кеш ОС). Если сервер перезагрузится спонтанно, новый слитый кусок может испортиться или потеряться. -Стандартное значение Linux dirty_expire_centisecs - 30 секунд (максимальное время, которое записанные данные хранятся только в оперативной памяти), но при больших нагрузках на дисковую систему, данные могут быть записаны намного позже. Экспериментально было найдено время - 480 секунд, за которое гарантированно новый кусок будет записан на диск. +Во время запуска сервер ClickHouse проверяет целостность кусков. +Если новый (слитый) кусок поврежден, ClickHouse возвращает неактивные куски в список активных и позже снова выполняет слияние. В этом случае испорченный кусок получает новое имя (добавляется префикс `broken_`) и попадает в каталог `detached`. +Если проверка целостности не выявляет проблем в слитом куске, то исходные неактивные куски переименовываются (добавляется префикс `ignored_`) и перемещаются в каталог `detached`. + +Стандартное для Linux значение `dirty_expire_centisecs` — 30 секунд. Это максимальное время, в течение которого записанные данные хранятся только в оперативной памяти. Если нагрузка на дисковую систему большая, то данные записываются намного позже. Значение 480 секунд подобрали экспериментальным путем — это время, за которое новый кусок гарантированно запишется на диск. ## replicated_fetches_http_connection_timeout {#replicated_fetches_http_connection_timeout} @@ -197,8 +203,8 @@ Eсли суммарное число активных кусков во все ## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool} -Максимальный суммарный размер кусков (в байтах) в одном слиянии, при наличии свободных ресурсов в фоновом пуле. -`max_bytes_to_merge_at_max_space_in_pool` -- примерно соответствует максимально возможному размеру куска, созданного автоматическим фоновым слиянием. +Максимальный суммарный размер кусков (в байтах) в одном слиянии, если есть свободные ресурсы в фоновом пуле. +`max_bytes_to_merge_at_max_space_in_pool` примерно соответствует максимально возможному размеру куска, созданного автоматическим фоновым слиянием. Возможные значения: @@ -206,26 +212,27 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 161061273600 (150ГБ). -Планировщик мержей периодически анализирует размер и количество кусков в партициях, и при достаточном количестве свободных ресурсов в фоновом пуле начинает фоновое слияние. Слияния происходят до тех пор, пока суммарный размер входных кусков не достигнет `max_bytes_to_merge_at_max_space_in_pool`. +Планировщик слияний периодически анализирует размер и количество кусков в партициях, и если в пуле хватает ресурсов, то начинает фоновое слияние. Слияния выполняются до тех пор, пока суммарный размер входных кусков не достигнет `max_bytes_to_merge_at_max_space_in_pool`. -Слияния, инициированные `optimize final`, не учитывают `max_bytes_to_merge_at_max_space_in_pool` и размеры кусков и слияют куски только с учетом наличия ресурсов в фоновом пуле, пока не останется один кусок в партиции. +Слияния, начатые по [OPTIMIZE FINAL](../../sql-reference/statements/optimize.md), не учитывают `max_bytes_to_merge_at_max_space_in_pool` и объединяют куски пока есть доступные ресурсы (свободное дисковое пространство) до тех пор, пока в партиции не останется один кусок. ## max_bytes_to_merge_at_min_space_in_pool {#max-bytes-to-merge-at-min-space-in-pool} -Максимальный суммарный размер кусков (в байтах) в одном слиянии, при минимальных свободных ресурсах в фоновом пуле. +Максимальный суммарный размер кусков (в байтах) в одном слиянии при минимуме свободных ресурсов в фоновом пуле. Возможные значения: - Положительное целое число. -Значение по умолчанию: 1048576 +Значение по умолчанию: 1048576 (1 МБ). -`max_bytes_to_merge_at_min_space_in_pool` задает максимальный суммарный размер кусков, для которых можно начать слияние, несмотря на недостаток свободных ресурсов в фоновом пуле (дискового пространства). Это необходимо, чтобы уменьшить количество маленьких кусков и вероятность ошибки `Too many parts`. -Слияния резервируют дисковое пространство, удваивая суммарный размер кусков в слиянии. Таким образом, при малом количестве свободного места на диске может сложится ситуация, что свободное место есть, но оно уже зарезервировано идущими слиянияними, поэтому другие слияния не могут начаться, и количество маленьких кусков в партиции растет с каждым инсертом. +`max_bytes_to_merge_at_min_space_in_pool` задает максимальный суммарный размер кусков, которые можно объединить несмотря на нехватку свободных ресурсов (дискового пространства) в фоновом пуле. Это нужно, чтобы уменьшить количество маленьких кусков и снизить вероятность ошибки `Too many parts`. + +Слияния резервируют дисковое пространство, удваивая суммарный размер кусков в слиянии. Поэтому при малом объеме свободного места на диске может сложиться ситуация, когда свободное место есть, но оно уже зарезервировано текущими слияниями. Из-за этого другие слияния не начинаются, и количество маленьких кусков в партиции растет с каждым запросом `INSERT`. ## merge_max_block_size {#merge-max-block-size} -Количество строк в блоках, которые читаются из слияемых кусков. +Количество строк в блоках, которые читаются из объединяемых кусков. Возможные значения: @@ -233,7 +240,7 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: 8192 -Слияние читает строки из кусков блоками по `merge_max_block_size` строк, производит слияние и пишет результат в новый кусок. Читаемый блок помещается в оперативную память, т.е. `merge_max_block_size` влияет на размер оперативной памяти, необходимой для слияния. Таким образом, слияния могут потреблять большое количество оперативной памяти для таблиц, хранящих очень большие строки (если средний размер строки 100кб, то при слиянии 10 кусков будет использовано (100кб * 10 * 8192) =~ 8ГБ ОЗУ). Уменьшив `merge_max_block_size`, можно сократить размер оперативной памяти, необходимой для слияния. +Слияние читает строки из кусков блоками по `merge_max_block_size` строк, производит слияние и записывает результат в новый кусок. Читаемый блок помещается в оперативную память, т.е. `merge_max_block_size` влияет на размер оперативной памяти, необходимой для слияния. Таким образом, слияния могут потреблять большое количество оперативной памяти для таблиц, хранящих очень большие строки (если средний размер строки 100кб, то при слиянии 10 кусков будет использовано (100кб * 10 * 8192) =~ 8ГБ оперативной памяти). Уменьшив `merge_max_block_size`, можно сократить размер оперативной памяти, необходимой для слияния, но при этом процесс слияния замедлится. ## max_part_loading_threads {#max-part-loading-threads} @@ -243,9 +250,9 @@ Eсли суммарное число активных кусков во все - Положительное целое число. -Значение по умолчанию: auto (количество ядер процессора). +Значение по умолчанию: определяется автоматически (по количеству ядер процессора). -При старте ClickHouse читает все куски всех таблиц (читает файлы с метаданными кусков), чтобы построить в ОЗУ список всех кусков. В некоторых системах с большим количеством кусков этот процесс может занимать длительное время, и это время можно сократить, увеличив `max_part_loading_threads` (если при этом процессе есть недозагруженность CPU и диска). +На старте ClickHouse читает все куски из всех таблиц (читает файлы с метаданными кусков), чтобы построить в оперативной памяти список всех кусков. В некоторых системах с большим количеством кусков этот процесс может занимать длительное время. Это время можно сократить, увеличив `max_part_loading_threads` (если при этом хватает ресурсов процессора и диска). ## max_partitions_to_read {#max-partitions-to-read} diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 6d39bab4990..29d184f6c34 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -362,6 +362,79 @@ SHOW [CURRENT] QUOTA SHOW ACCESS ``` +## SHOW CLUSTER(s) {#show-cluster-statement} + +Возвращает список кластеров. Все доступные кластеры перечислены в таблице [system.clusters](../../operations/system-tables/clusters.md). + +!!! info "Note" + По запросу `SHOW CLUSTER name` вы получите содержимое таблицы system.clusters для этого кластера. + +### Синтаксис {#show-cluster-syntax} + +``` sql +SHOW CLUSTER '' +SHOW CLUSTERS [LIKE|NOT LIKE ''] [LIMIT ] +``` +### Примеры {#show-cluster-examples} + +Запрос: + +``` sql +SHOW CLUSTERS; +``` + +Результат: + +```text +┌─cluster──────────────────────────────────────┐ +│ test_cluster_two_shards │ +│ test_cluster_two_shards_internal_replication │ +│ test_cluster_two_shards_localhost │ +│ test_shard_localhost │ +│ test_shard_localhost_secure │ +│ test_unavailable_shard │ +└──────────────────────────────────────────────┘ +``` + +Запрос: + +``` sql +SHOW CLUSTERS LIKE 'test%' LIMIT 1; +``` + +Результат: + +```text +┌─cluster─────────────────┐ +│ test_cluster_two_shards │ +└─────────────────────────┘ +``` + +Запрос: + +``` sql +SHOW CLUSTER 'test_shard_localhost' FORMAT Vertical; +``` + +Результат: + +```text +Row 1: +────── +cluster: test_shard_localhost +shard_num: 1 +shard_weight: 1 +replica_num: 1 +host_name: localhost +host_address: 127.0.0.1 +port: 9000 +is_local: 1 +user: default +default_database: +errors_count: 0 +estimated_recovery_time: 0 +``` + ## SHOW SETTINGS {#show-settings} Возвращает список системных настроек и их значений. Использует данные из таблицы [system.settings](../../operations/system-tables/settings.md). @@ -426,4 +499,3 @@ SHOW CHANGED SETTINGS ILIKE '%MEMORY%' **См. также** - Таблица [system.settings](../../operations/system-tables/settings.md) - From 7c81b377484de29f2c01f233a8c391a44e3aa8e1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 15 Jun 2021 15:25:34 +0300 Subject: [PATCH 195/491] Update libpqxx --- contrib/libpqxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libpqxx b/contrib/libpqxx index 1e29e01d794..78f4488db02 160000 --- a/contrib/libpqxx +++ b/contrib/libpqxx @@ -1 +1 @@ -Subproject commit 1e29e01d7943fa1481e3694050e03bdddfa38703 +Subproject commit 78f4488db0266d15ab0af5a8a3791065916e1cc8 From 235e3e2f5ba2e6061fbeac1e282f0fc9f7d3b9b5 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 20 May 2021 16:47:12 +0300 Subject: [PATCH 196/491] Support structs in Arrow/Parquet/ORC --- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 48 ++++++++++++- .../Formats/Impl/CHColumnToArrowColumn.cpp | 51 ++++++++++++- .../Formats/Impl/ORCBlockInputFormat.cpp | 2 +- .../Formats/Impl/ORCBlockOutputFormat.cpp | 71 +++++++++++++++---- .../Formats/Impl/ORCBlockOutputFormat.h | 4 +- .../Formats/Impl/ParquetBlockInputFormat.cpp | 23 +++++- .../00900_orc_tuples_load.reference | 2 + .../0_stateless/00900_orc_tuples_load.sh | 17 +++++ .../00900_parquet_tuples_load.reference | 2 + .../0_stateless/00900_parquet_tuples_load.sh | 17 +++++ .../0_stateless/01273_arrow_arrays_load.sh | 2 +- .../01273_arrow_tuples_load.reference | 2 + .../0_stateless/01273_arrow_tuples_load.sh | 17 +++++ 13 files changed, 237 insertions(+), 21 deletions(-) create mode 100644 tests/queries/0_stateless/00900_orc_tuples_load.reference create mode 100755 tests/queries/0_stateless/00900_orc_tuples_load.sh create mode 100644 tests/queries/0_stateless/00900_parquet_tuples_load.reference create mode 100755 tests/queries/0_stateless/00900_parquet_tuples_load.sh create mode 100644 tests/queries/0_stateless/01273_arrow_tuples_load.reference create mode 100755 tests/queries/0_stateless/01273_arrow_tuples_load.sh diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 29fff1fd4e0..23f356a77fc 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -7,15 +7,17 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include #include #include -#include namespace DB @@ -309,8 +311,6 @@ namespace DB break; case arrow::Type::LIST: { - const auto * list_type = static_cast(arrow_column->type().get()); - auto list_nested_type = list_type->value_type(); arrow::ArrayVector array_vector; array_vector.reserve(arrow_column->num_chunks()); for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) @@ -326,6 +326,25 @@ namespace DB fillOffsetsFromArrowListColumn(arrow_column, column_array.getOffsetsColumn()); break; } + case arrow::Type::STRUCT: + { + ColumnTuple & column_tuple = typeid_cast(internal_column); + int fields_count = column_tuple.tupleSize(); + std::vector nested_arrow_columns(fields_count); + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::StructArray & struct_chunk = static_cast(*(arrow_column->chunk(chunk_i))); + for (int i = 0; i < fields_count; ++i) + nested_arrow_columns[i].emplace_back(struct_chunk.field(i)); + } + + for (int i = 0; i != fields_count; ++i) + { + auto nested_arrow_column = std::make_shared(nested_arrow_columns[i]); + readColumnFromArrowColumn(nested_arrow_column, column_tuple.getColumn(i), column_name, format_name, false); + } + break; + } # define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ case ARROW_NUMERIC_TYPE: \ fillColumnWithNumericData(arrow_column, internal_column); \ @@ -372,6 +391,29 @@ namespace DB return std::make_shared(getInternalType(list_nested_type, array_type->getNestedType(), column_name, format_name)); } + if (arrow_type->id() == arrow::Type::STRUCT) + { + const auto * struct_type = static_cast(arrow_type.get()); + const DataTypeTuple * tuple_type = typeid_cast(column_type.get()); + if (!tuple_type) + throw Exception{"Cannot convert arrow STRUCT type to a not Tuple ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + + const DataTypes & tuple_nested_types = tuple_type->getElements(); + int internal_fields_num = tuple_nested_types.size(); + /// If internal column has less elements then arrow struct, we will select only first internal_fields_num columns. + if (internal_fields_num > struct_type->num_fields()) + throw Exception{ + "Cannot convert arrow STRUCT with " + std::to_string(struct_type->num_fields()) + " fields to a ClickHouse Tuple with " + + std::to_string(internal_fields_num) + " elements " + column_type->getName(), + ErrorCodes::CANNOT_CONVERT_TYPE}; + + DataTypes nested_types; + for (int i = 0; i < internal_fields_num; ++i) + nested_types.push_back(getInternalType(struct_type->field(i)->type(), tuple_nested_types[i], column_name, format_name)); + + return std::make_shared(std::move(nested_types)); + } + if (const auto * internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(), [=](auto && elem) { return elem.first == arrow_type->id(); }); internal_type_it != arrow_type_to_internal_type.end()) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 0e9968bec17..4626d87f0cc 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -6,11 +6,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -113,7 +115,7 @@ namespace DB size_t start, size_t end) { - const auto * column_array = static_cast(column.get()); + const auto * column_array = typeid_cast(column.get()); ColumnPtr nested_column = column_array->getDataPtr(); DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); const auto & offsets = column_array->getOffsets(); @@ -124,13 +126,41 @@ namespace DB for (size_t array_idx = start; array_idx < end; ++array_idx) { - /// Start new array + /// Start new array. components_status = builder.Append(); checkStatus(components_status, nested_column->getName(), format_name); fillArrowArray(column_name, nested_column, nested_type, null_bytemap, value_builder, format_name, offsets[array_idx - 1], offsets[array_idx]); } } + static void fillArrowArrayWithTupleColumnData( + const String & column_name, + ColumnPtr & column, + const std::shared_ptr & column_type, + const PaddedPODArray * null_bytemap, + arrow::ArrayBuilder * array_builder, + String format_name, + size_t start, + size_t end) + { + const auto * column_tuple = typeid_cast(column.get()); + const auto & nested_types = typeid_cast(column_type.get())->getElements(); + + arrow::StructBuilder & builder = assert_cast(*array_builder); + + for (size_t i = 0; i != column_tuple->tupleSize(); ++i) + { + ColumnPtr nested_column = column_tuple->getColumnPtr(i); + fillArrowArray(column_name + "." + std::to_string(i), nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end); + } + + for (size_t i = start; i != end; ++i) + { + auto status = builder.Append(); + checkStatus(status, column->getName(), format_name); + } + } + template static void fillArrowArrayWithStringColumnData( ColumnPtr write_column, @@ -251,6 +281,10 @@ namespace DB { fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end); } + else if ("Tuple" == column_type_name) + { + fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end); + } else if (isDecimal(column_type)) { auto fill_decimal = [&](const auto & types) -> bool @@ -351,6 +385,19 @@ namespace DB return arrow::list(nested_arrow_type); } + if (isTuple(column_type)) + { + const auto & nested_types = typeid_cast(column_type.get())->getElements(); + std::vector> nested_fields; + for (size_t i = 0; i != nested_types.size(); ++i) + { + String name = column_name + "." + std::to_string(i); + auto nested_arrow_type = getArrowType(nested_types[i], name, format_name, is_column_nullable); + nested_fields.push_back(std::make_shared(name, nested_arrow_type, *is_column_nullable)); + } + return arrow::struct_(std::move(nested_fields)); + } + const std::string type_name = column_type->getFamilyName(); if (const auto * arrow_type_it = std::find_if( internal_type_to_arrow_type.begin(), diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 24b524faeaf..572b1c130e3 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -67,7 +67,7 @@ void ORCBlockInputFormat::resetParser() stripe_current = 0; } -size_t countIndicesForType(std::shared_ptr type) +static size_t countIndicesForType(std::shared_ptr type) { if (type->id() == arrow::Type::LIST) return countIndicesForType(static_cast(type.get())->value_type()) + 1; diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index ec6a7a65573..fa6f386adf1 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -10,12 +10,14 @@ #include #include #include +#include #include #include #include #include #include +#include namespace DB { @@ -46,15 +48,9 @@ void ORCOutputStream::write(const void* buf, size_t length) ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_), format_settings{format_settings_}, output_stream(out_), data_types(header_.getDataTypes()) { - schema = orc::createStructType(); - options.setCompression(orc::CompressionKind::CompressionKind_NONE); - size_t columns_count = header_.columns(); - for (size_t i = 0; i != columns_count; ++i) - schema->addStructField(header_.safeGetByPosition(i).name, getORCType(data_types[i])); - writer = orc::createWriter(*schema, &output_stream, options); } -ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & type) +ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const std::string & column_name) { switch (type->getTypeId()) { @@ -102,12 +98,12 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t } case TypeIndex::Nullable: { - return getORCType(removeNullable(type)); + return getORCType(removeNullable(type), column_name); } case TypeIndex::Array: { const auto * array_type = typeid_cast(type.get()); - return orc::createListType(getORCType(array_type->getNestedType())); + return orc::createListType(getORCType(array_type->getNestedType(), column_name)); } case TypeIndex::Decimal32: { @@ -124,6 +120,18 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t const auto * decimal_type = typeid_cast *>(type.get()); return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); } + case TypeIndex::Tuple: + { + const auto * tuple_type = typeid_cast(type.get()); + const auto & nested_types = tuple_type->getElements(); + auto struct_type = orc::createStructType(); + for (size_t i = 0; i < nested_types.size(); ++i) + { + String name = column_name + "." + std::to_string(i); + struct_type->addStructField(name, getORCType(nested_types[i], name)); + } + return struct_type; + } default: { throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN); @@ -149,6 +157,8 @@ void ORCBlockOutputFormat::writeNumbers( number_orc_column.notNull[i] = 0; continue; } + else + number_orc_column.notNull[i] = 1; number_orc_column.data[i] = convert(number_column.getElement(i)); } number_orc_column.numElements = number_column.size(); @@ -175,6 +185,9 @@ void ORCBlockOutputFormat::writeDecimals( decimal_orc_column.notNull[i] = 0; continue; } + else + decimal_orc_column.notNull[i] = 1; + decimal_orc_column.values[i] = convert(decimal_column.getElement(i).value); } decimal_orc_column.numElements = decimal_column.size(); @@ -197,6 +210,9 @@ void ORCBlockOutputFormat::writeStrings( string_orc_column.notNull[i] = 0; continue; } + else + string_orc_column.notNull[i] = 1; + const StringRef & string = string_column.getDataAt(i); string_orc_column.data[i] = const_cast(string.data); string_orc_column.length[i] = string.size; @@ -223,6 +239,9 @@ void ORCBlockOutputFormat::writeDateTimes( timestamp_orc_column.notNull[i] = 0; continue; } + else + timestamp_orc_column.notNull[i] = 1; + timestamp_orc_column.data[i] = get_seconds(timestamp_column.getElement(i)); timestamp_orc_column.nanoseconds[i] = get_nanoseconds(timestamp_column.getElement(i)); } @@ -235,11 +254,10 @@ void ORCBlockOutputFormat::writeColumn( DataTypePtr & type, const PaddedPODArray * null_bytemap) { + orc_column.notNull.resize(column.size()); if (null_bytemap) - { orc_column.hasNulls = true; - orc_column.notNull.resize(column.size()); - } + switch (type->getTypeId()) { case TypeIndex::Int8: @@ -374,12 +392,25 @@ void ORCBlockOutputFormat::writeColumn( for (size_t i = 0; i != list_column.size(); ++i) { list_orc_column.offsets[i + 1] = offsets[i]; + list_orc_column.notNull[i] = 1; } orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements; writeColumn(nested_orc_column, list_column.getData(), nested_type, null_bytemap); list_orc_column.numElements = list_column.size(); break; } + case TypeIndex::Tuple: + { + orc::StructVectorBatch & struct_orc_column = dynamic_cast(orc_column); + const auto & tuple_column = assert_cast(column); + auto nested_types = typeid_cast(type.get())->getElements(); + for (size_t i = 0; i != tuple_column.size(); ++i) + struct_orc_column.notNull[i] = 1; + for (size_t i = 0; i != tuple_column.tupleSize(); ++i) + writeColumn(*struct_orc_column.fields[i], tuple_column.getColumn(i), nested_types[i], null_bytemap); + break; + + } default: throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN); } @@ -409,6 +440,8 @@ size_t ORCBlockOutputFormat::getMaxColumnSize(Chunk & chunk) void ORCBlockOutputFormat::consume(Chunk chunk) { + if (!writer) + prepareWriter(); size_t columns_num = chunk.getNumColumns(); size_t rows_num = chunk.getNumRows(); /// getMaxColumnSize is needed to write arrays. @@ -425,9 +458,23 @@ void ORCBlockOutputFormat::consume(Chunk chunk) void ORCBlockOutputFormat::finalize() { + if (!writer) + prepareWriter(); + writer->close(); } +void ORCBlockOutputFormat::prepareWriter() +{ + const Block & header = getPort(PortKind::Main).getHeader(); + schema = orc::createStructType(); + options.setCompression(orc::CompressionKind::CompressionKind_NONE); + size_t columns_count = header.columns(); + for (size_t i = 0; i != columns_count; ++i) + schema->addStructField(header.safeGetByPosition(i).name, getORCType(data_types[i], header.safeGetByPosition(i).name)); + writer = orc::createWriter(*schema, &output_stream, options); +} + void registerOutputFormatProcessorORC(FormatFactory & factory) { factory.registerOutputFormatProcessor("ORC", []( diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index 05053317533..557bf6cc07a 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -43,7 +43,7 @@ public: void finalize() override; private: - ORC_UNIQUE_PTR getORCType(const DataTypePtr & type); + ORC_UNIQUE_PTR getORCType(const DataTypePtr & type, const std::string & column_name); /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be /// converted to unsigned char (bugprone-signed-char-misuse in clang). @@ -71,6 +71,8 @@ private: size_t getColumnSize(const IColumn & column, DataTypePtr & type); size_t getMaxColumnSize(Chunk & chunk); + void prepareWriter(); + const FormatSettings format_settings; ORCOutputStream output_stream; DataTypes data_types; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index df264406cfe..969aaf6ff36 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -67,6 +67,24 @@ void ParquetBlockInputFormat::resetParser() row_group_current = 0; } +static size_t countIndicesForType(std::shared_ptr type) +{ + if (type->id() == arrow::Type::LIST) + return countIndicesForType(static_cast(type.get())->value_type()); + + int indices = 0; + if (type->id() == arrow::Type::STRUCT) + { + auto * struct_type = static_cast(type.get()); + for (int i = 0; i != struct_type->num_fields(); ++i) + indices += countIndicesForType(struct_type->field(i)->type()); + } + else + indices = 1; + + return indices; +} + void ParquetBlockInputFormat::prepareReader() { THROW_ARROW_NOT_OK(parquet::arrow::OpenFile(asArrowFile(in), arrow::default_memory_pool(), &file_reader)); @@ -76,11 +94,14 @@ void ParquetBlockInputFormat::prepareReader() std::shared_ptr schema; THROW_ARROW_NOT_OK(file_reader->GetSchema(&schema)); + int index = 0; for (int i = 0; i < schema->num_fields(); ++i) { if (getPort().getHeader().has(schema->field(i)->name())) { - column_indices.push_back(i); + int indexes_count = countIndicesForType(schema->field(i)->type()); + for (int j = 0; j != indexes_count; ++j) + column_indices.push_back(index++); } } } diff --git a/tests/queries/0_stateless/00900_orc_tuples_load.reference b/tests/queries/0_stateless/00900_orc_tuples_load.reference new file mode 100644 index 00000000000..cbf8ee819f7 --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_tuples_load.reference @@ -0,0 +1,2 @@ +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/00900_orc_tuples_load.sh b/tests/queries/0_stateless/00900_orc_tuples_load.sh new file mode 100755 index 00000000000..f32be6af751 --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_tuples_load.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_tuples" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_tuples FORMAT ORC" > "${CLICKHOUSE_TMP}"/tuples.orc + +cat "${CLICKHOUSE_TMP}"/tuples.orc | ${CLICKHOUSE_CLIENT} -q "INSERT INTO orc_tuples FORMAT ORC" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_tuples" +${CLICKHOUSE_CLIENT} --query="DROP TABLE orc_tuples" diff --git a/tests/queries/0_stateless/00900_parquet_tuples_load.reference b/tests/queries/0_stateless/00900_parquet_tuples_load.reference new file mode 100644 index 00000000000..cbf8ee819f7 --- /dev/null +++ b/tests/queries/0_stateless/00900_parquet_tuples_load.reference @@ -0,0 +1,2 @@ +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/00900_parquet_tuples_load.sh b/tests/queries/0_stateless/00900_parquet_tuples_load.sh new file mode 100755 index 00000000000..031e8111019 --- /dev/null +++ b/tests/queries/0_stateless/00900_parquet_tuples_load.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_tuples" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_tuples FORMAT Parquet" > "${CLICKHOUSE_TMP}"/tuples.parquet + +cat "${CLICKHOUSE_TMP}"/tuples.parquet | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_tuples FORMAT Parquet" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_tuples" +${CLICKHOUSE_CLIENT} --query="DROP TABLE parquet_tuples" diff --git a/tests/queries/0_stateless/01273_arrow_arrays_load.sh b/tests/queries/0_stateless/01273_arrow_arrays_load.sh index b8d1a85921d..bce653376a5 100755 --- a/tests/queries/0_stateless/01273_arrow_arrays_load.sh +++ b/tests/queries/0_stateless/01273_arrow_arrays_load.sh @@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_arrays" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_arrays" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_arrays (arr1 Array(Int8), arr2 Array(UInt8), arr3 Array(Int16), arr4 Array(UInt16), arr5 Array(Int32), arr6 Array(UInt32), arr7 Array(Int64), arr8 Array(UInt64), arr9 Array(String), arr10 Array(FixedString(4)), arr11 Array(Float32), arr12 Array(Float64), arr13 Array(Date), arr14 Array(Datetime)) ENGINE=Memory()" ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_arrays VALUES ([1,-2,3],[1,2,3],[100,-200,300],[100,200,300],[10000000,-20000000,30000000],[10000000,2000000,3000000],[100000000000000,-200000000000,3000000000000],[100000000000000,20000000000000,3000000000000],['Some string','Some string','Some string'],['0000','1111','2222'],[42.42,424.2,0.4242],[424242.424242,4242042420.242424,42],['2000-01-01','2001-01-01','2002-01-01'],['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']),([],[],[],[],[],[],[],[],[],[],[],[],[],[])" diff --git a/tests/queries/0_stateless/01273_arrow_tuples_load.reference b/tests/queries/0_stateless/01273_arrow_tuples_load.reference new file mode 100644 index 00000000000..cbf8ee819f7 --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_tuples_load.reference @@ -0,0 +1,2 @@ +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/01273_arrow_tuples_load.sh b/tests/queries/0_stateless/01273_arrow_tuples_load.sh new file mode 100755 index 00000000000..311079afe50 --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_tuples_load.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_tuples" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_tuples FORMAT Arrow" > "${CLICKHOUSE_TMP}"/tuples.arrow + +cat "${CLICKHOUSE_TMP}"/tuples.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_tuples FORMAT Arrow" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_tuples" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_tuples" From d74c5ef62d02a91028d4f4152b358c2c837515a2 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 20 May 2021 17:02:44 +0300 Subject: [PATCH 197/491] Minor change --- .../Formats/Impl/ORCBlockOutputFormat.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index fa6f386adf1..a560701944d 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -157,8 +157,8 @@ void ORCBlockOutputFormat::writeNumbers( number_orc_column.notNull[i] = 0; continue; } - else - number_orc_column.notNull[i] = 1; + + number_orc_column.notNull[i] = 1; number_orc_column.data[i] = convert(number_column.getElement(i)); } number_orc_column.numElements = number_column.size(); @@ -185,9 +185,8 @@ void ORCBlockOutputFormat::writeDecimals( decimal_orc_column.notNull[i] = 0; continue; } - else - decimal_orc_column.notNull[i] = 1; + decimal_orc_column.notNull[i] = 1; decimal_orc_column.values[i] = convert(decimal_column.getElement(i).value); } decimal_orc_column.numElements = decimal_column.size(); @@ -210,9 +209,8 @@ void ORCBlockOutputFormat::writeStrings( string_orc_column.notNull[i] = 0; continue; } - else - string_orc_column.notNull[i] = 1; + string_orc_column.notNull[i] = 1; const StringRef & string = string_column.getDataAt(i); string_orc_column.data[i] = const_cast(string.data); string_orc_column.length[i] = string.size; @@ -239,9 +237,8 @@ void ORCBlockOutputFormat::writeDateTimes( timestamp_orc_column.notNull[i] = 0; continue; } - else - timestamp_orc_column.notNull[i] = 1; + timestamp_orc_column.notNull[i] = 1; timestamp_orc_column.data[i] = get_seconds(timestamp_column.getElement(i)); timestamp_orc_column.nanoseconds[i] = get_nanoseconds(timestamp_column.getElement(i)); } From 540c494492add6999fdb3e956fd00fdc821602d0 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Mon, 24 May 2021 14:39:08 +0300 Subject: [PATCH 198/491] Fix selecting indexes in ORC and Parquet formats --- src/Processors/Formats/Impl/ORCBlockInputFormat.cpp | 13 ++++++++----- .../Formats/Impl/ParquetBlockInputFormat.cpp | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 572b1c130e3..d16d0870474 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -84,17 +84,20 @@ void ORCBlockInputFormat::prepareReader() std::shared_ptr schema; THROW_ARROW_NOT_OK(file_reader->ReadSchema(&schema)); - int index = 0; + /// In ReadStripe column indices should be started from 1, + /// because 0 indicates to select all columns. + int index = 1; for (int i = 0; i < schema->num_fields(); ++i) { + /// LIST type require 2 indices, so we should recursively + /// count the number of indices we need for this type. + int indexes_count = countIndicesForType(schema->field(i)->type()); if (getPort().getHeader().has(schema->field(i)->name())) { - /// LIST type require 2 indices, so we should recursively - /// count the number of indices we need for this type. - int indexes_count = countIndicesForType(schema->field(i)->type()); for (int j = 0; j != indexes_count; ++j) - include_indices.push_back(index++); + include_indices.push_back(index + j); } + index += indexes_count; } } diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 969aaf6ff36..0223e5219c0 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -97,12 +97,16 @@ void ParquetBlockInputFormat::prepareReader() int index = 0; for (int i = 0; i < schema->num_fields(); ++i) { + /// STRUCT type require the number of indexes equal to the number of + /// nested elements, so we should recursively + /// count the number of indices we need for this type. + int indexes_count = countIndicesForType(schema->field(i)->type()); if (getPort().getHeader().has(schema->field(i)->name())) { - int indexes_count = countIndicesForType(schema->field(i)->type()); for (int j = 0; j != indexes_count; ++j) - column_indices.push_back(index++); + column_indices.push_back(index + j); } + index += indexes_count; } } From c8b37977da865ad7c3cd0f22b9a9f5a0944ab6d6 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 25 May 2021 15:01:28 +0300 Subject: [PATCH 199/491] Fix bugs, support dictionary for Arrow format --- src/Columns/ColumnLowCardinality.h | 1 + src/Core/Settings.h | 5 +- src/DataTypes/DataTypeLowCardinality.h | 2 +- src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 1 + .../Formats/Impl/ArrowBlockInputFormat.cpp | 2 +- .../Formats/Impl/ArrowBlockInputFormat.h | 3 + .../Formats/Impl/ArrowBlockOutputFormat.cpp | 2 +- .../Formats/Impl/ArrowBlockOutputFormat.h | 2 + .../Formats/Impl/ArrowColumnToCHColumn.cpp | 77 ++++++- .../Formats/Impl/ArrowColumnToCHColumn.h | 16 +- .../Formats/Impl/CHColumnToArrowColumn.cpp | 197 ++++++++++++++++-- .../Formats/Impl/CHColumnToArrowColumn.h | 21 +- .../Formats/Impl/ORCBlockInputFormat.cpp | 16 +- .../Formats/Impl/ORCBlockInputFormat.h | 3 + .../Formats/Impl/ParquetBlockInputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockInputFormat.h | 2 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.h | 2 + .../01273_arrow_dictionaries_load.reference | 4 + .../01273_arrow_dictionaries_load.sh | 24 +++ 21 files changed, 346 insertions(+), 39 deletions(-) create mode 100644 tests/queries/0_stateless/01273_arrow_dictionaries_load.reference create mode 100755 tests/queries/0_stateless/01273_arrow_dictionaries_load.sh diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index fc607021ccf..b39e4cf79c0 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -190,6 +190,7 @@ public: void nestedToNullable() { dictionary.getColumnUnique().nestedToNullable(); } const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); } + IColumnUnique & getDictionary() { return dictionary.getColumnUnique(); } const ColumnPtr & getDictionaryPtr() const { return dictionary.getColumnUniquePtr(); } /// IColumnUnique & getUnique() { return static_cast(*column_unique); } /// ColumnPtr getUniquePtr() const { return column_unique; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2aed174c088..4815f060140 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -117,7 +117,7 @@ class IColumn; M(Bool, skip_unavailable_shards, false, "If 1, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ - M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ + M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards. If 2 - same as 1 but also apply ORDER BY and LIMIT stages", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ @@ -559,7 +559,8 @@ class IColumn; M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \ M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \ M(Bool, cross_to_inner_join_rewrite, true, "Use inner join instead of comma/cross join if possible", 0) \ - + \ + M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/DataTypeLowCardinality.h index 1266174c6d6..2ab62fabf41 100644 --- a/src/DataTypes/DataTypeLowCardinality.h +++ b/src/DataTypes/DataTypeLowCardinality.h @@ -13,7 +13,7 @@ private: DataTypePtr dictionary_type; public: - DataTypeLowCardinality(DataTypePtr dictionary_type_); + DataTypeLowCardinality(DataTypePtr dictionary_type_dict); const DataTypePtr & getDictionaryType() const { return dictionary_type; } diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index f1f60ae2ac4..8b7cf9635b4 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -112,6 +112,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; format_settings.with_names_use_header = settings.input_format_with_names_use_header; format_settings.write_statistics = settings.output_format_write_statistics; + format_settings.arrow.low_cardinality_as_dictionary = settings.output_format_arrow_low_cardinality_as_dictionary; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context if (format_settings.schema.is_server) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 33d51b1797f..1773f2cc2c6 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -52,6 +52,7 @@ struct FormatSettings struct { UInt64 row_group_size = 1000000; + bool low_cardinality_as_dictionary = false; } arrow; struct diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 52d2cf98c25..0ac8251b8bb 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -63,7 +63,7 @@ Chunk ArrowBlockInputFormat::generate() ++record_batch_current; - ArrowColumnToCHColumn::arrowTableToCHChunk(res, *table_result, header, "Arrow"); + arrow_column_to_ch_column.arrowTableToCHChunk(res, *table_result, header, "Arrow"); return res; } diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 5ad112efde9..6a8acc4a118 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -3,6 +3,7 @@ #if USE_ARROW #include +#include namespace arrow { class RecordBatchReader; } namespace arrow::ipc { class RecordBatchFileReader; } @@ -32,6 +33,8 @@ private: // The following fields are used only for Arrow format std::shared_ptr file_reader; + ArrowColumnToCHColumn arrow_column_to_ch_column; + int record_batch_total = 0; int record_batch_current = 0; diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index c1abdd1a759..d1fdffb700c 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -28,7 +28,7 @@ void ArrowBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow"); + ch_column_to_arrow_column.chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); if (!writer) prepareWriter(arrow_table->schema()); diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h index 0cc6804705b..fc8efe62435 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h @@ -4,6 +4,7 @@ #include #include +#include #include "ArrowBufferedStreams.h" namespace arrow { class Schema; } @@ -28,6 +29,7 @@ private: const FormatSettings format_settings; std::shared_ptr arrow_ostream; std::shared_ptr writer; + CHColumnToArrowColumn ch_column_to_arrow_column; void prepareWriter(const std::shared_ptr & schema); }; diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 23f356a77fc..23987067474 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -266,12 +268,33 @@ namespace DB } } - static void readColumnFromArrowColumn(std::shared_ptr & arrow_column, IColumn & internal_column, const std::string & column_name, const std::string format_name, bool is_nullable) + static DataTypePtr getInternalIndexesType(std::shared_ptr arrow_type) + { + switch (arrow_type->id()) + { +# define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ + case ARROW_NUMERIC_TYPE: \ + return std::make_shared>(); \ + + FOR_ARROW_INXEXES_TYPES(DISPATCH) +# undef DISPATCH + default: + throw Exception("Unsupported type for indexes in LowCardinality: " + arrow_type->name() + ".", ErrorCodes::BAD_ARGUMENTS); + } + } + + static void readColumnFromArrowColumn( + std::shared_ptr & arrow_column, + IColumn & internal_column, + const std::string & column_name, + const std::string format_name, + bool is_nullable, + std::unordered_map dictionary_values) { if (internal_column.isNullable()) { ColumnNullable & column_nullable = typeid_cast(internal_column); - readColumnFromArrowColumn(arrow_column, column_nullable.getNestedColumn(), column_name, format_name, true); + readColumnFromArrowColumn(arrow_column, column_nullable.getNestedColumn(), column_name, format_name, true, dictionary_values); fillByteMapFromArrowColumn(arrow_column, column_nullable.getNullMapColumn()); return; } @@ -322,7 +345,7 @@ namespace DB auto arrow_nested_column = std::make_shared(array_vector); ColumnArray & column_array = typeid_cast(internal_column); - readColumnFromArrowColumn(arrow_nested_column, column_array.getData(), column_name, format_name, false); + readColumnFromArrowColumn(arrow_nested_column, column_array.getData(), column_name, format_name, false, dictionary_values); fillOffsetsFromArrowListColumn(arrow_column, column_array.getOffsetsColumn()); break; } @@ -341,10 +364,48 @@ namespace DB for (int i = 0; i != fields_count; ++i) { auto nested_arrow_column = std::make_shared(nested_arrow_columns[i]); - readColumnFromArrowColumn(nested_arrow_column, column_tuple.getColumn(i), column_name, format_name, false); + readColumnFromArrowColumn(nested_arrow_column, column_tuple.getColumn(i), column_name, format_name, false, dictionary_values); } break; } + case arrow::Type::DICTIONARY: + { + ColumnLowCardinality & column_lc = typeid_cast(internal_column); + auto & dict_values = dictionary_values[column_name]; + /// Load dictionary values only once and reuse it. + if (!dict_values) + { + arrow::ArrayVector dict_array; + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::DictionaryArray & dict_chunk = static_cast(*(arrow_column->chunk(chunk_i))); + dict_array.emplace_back(dict_chunk.dictionary()); + } + auto arrow_dict_column = std::make_shared(dict_array); + + auto dict_column = IColumn::mutate(column_lc.getDictionaryPtr()); + auto * uniq_column = static_cast(dict_column.get()); + auto values_column = uniq_column->getNestedColumn()->cloneEmpty(); + readColumnFromArrowColumn(arrow_dict_column, *values_column, column_name, format_name, false, dictionary_values); + uniq_column->uniqueInsertRangeFrom(*values_column, 0, values_column->size()); + dict_values = std::move(dict_column); + } + + arrow::ArrayVector indexes_array; + for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) + { + arrow::DictionaryArray & dict_chunk = static_cast(*(arrow_column->chunk(chunk_i))); + indexes_array.emplace_back(dict_chunk.indices()); + } + + auto arrow_indexes_column = std::make_shared(indexes_array); + auto indexes_column = getInternalIndexesType(arrow_indexes_column->type())->createColumn(); + + readColumnFromArrowColumn(arrow_indexes_column, *indexes_column, column_name, format_name, is_nullable, dictionary_values); + auto new_column_lc = ColumnLowCardinality::create(dict_values, std::move(indexes_column)); + column_lc = std::move(*new_column_lc); + break; + } # define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ case ARROW_NUMERIC_TYPE: \ fillColumnWithNumericData(arrow_column, internal_column); \ @@ -414,6 +475,12 @@ namespace DB return std::make_shared(std::move(nested_types)); } + if (arrow_type->id() == arrow::Type::DICTIONARY) + { + const auto * arrow_dict_type = static_cast(arrow_type.get()); + return std::make_shared(getInternalType(arrow_dict_type->value_type(), column_type, column_name, format_name)); + } + if (const auto * internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(), [=](auto && elem) { return elem.first == arrow_type->id(); }); internal_type_it != arrow_type_to_internal_type.end()) @@ -459,7 +526,7 @@ namespace DB DataTypePtr internal_type = getInternalType(arrow_column->type(), column_type, header_column.name, format_name); MutableColumnPtr read_column = internal_type->createColumn(); - readColumnFromArrowColumn(arrow_column, *read_column, header_column.name, format_name, false); + readColumnFromArrowColumn(arrow_column, *read_column, header_column.name, format_name, false, dictionary_values); ColumnWithTypeAndName column; column.name = header_column.name; diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index abac501c4c5..60fbc30e9c9 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -36,10 +36,24 @@ namespace DB M(arrow::Type::FLOAT, DB::Float32) \ M(arrow::Type::DOUBLE, DB::Float64) +# define FOR_ARROW_INXEXES_TYPES(M) \ + M(arrow::Type::UINT8, DB::UInt8) \ + M(arrow::Type::INT8, DB::UInt8) \ + M(arrow::Type::UINT16, DB::UInt16) \ + M(arrow::Type::INT16, DB::UInt16) \ + M(arrow::Type::UINT32, DB::UInt32) \ + M(arrow::Type::INT32, DB::UInt32) \ + M(arrow::Type::UINT64, DB::UInt64) \ + M(arrow::Type::INT64, DB::UInt64) \ + + /// Map {column name : dictionary column}. + /// To avoid converting dictionary from Arrow Dictionary + /// to LowCardinality every chunk we save it and reuse. + std::unordered_map dictionary_values; public: - static void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, + void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, const Block & header, std::string format_name); }; } diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 4626d87f0cc..97a5326a421 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -7,18 +7,19 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include #include -#include namespace DB @@ -103,7 +104,8 @@ namespace DB arrow::ArrayBuilder * array_builder, String format_name, size_t start, - size_t end); + size_t end, + std::unordered_map> & dictionary_values); static void fillArrowArrayWithArrayColumnData( const String & column_name, @@ -113,7 +115,8 @@ namespace DB arrow::ArrayBuilder * array_builder, String format_name, size_t start, - size_t end) + size_t end, + std::unordered_map> & dictionary_values) { const auto * column_array = typeid_cast(column.get()); ColumnPtr nested_column = column_array->getDataPtr(); @@ -129,7 +132,7 @@ namespace DB /// Start new array. components_status = builder.Append(); checkStatus(components_status, nested_column->getName(), format_name); - fillArrowArray(column_name, nested_column, nested_type, null_bytemap, value_builder, format_name, offsets[array_idx - 1], offsets[array_idx]); + fillArrowArray(column_name, nested_column, nested_type, null_bytemap, value_builder, format_name, offsets[array_idx - 1], offsets[array_idx], dictionary_values); } } @@ -141,7 +144,8 @@ namespace DB arrow::ArrayBuilder * array_builder, String format_name, size_t start, - size_t end) + size_t end, + std::unordered_map> & dictionary_values) { const auto * column_tuple = typeid_cast(column.get()); const auto & nested_types = typeid_cast(column_type.get())->getElements(); @@ -151,7 +155,7 @@ namespace DB for (size_t i = 0; i != column_tuple->tupleSize(); ++i) { ColumnPtr nested_column = column_tuple->getColumnPtr(i); - fillArrowArray(column_name + "." + std::to_string(i), nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end); + fillArrowArray(column_name + "." + std::to_string(i), nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end, dictionary_values); } for (size_t i = start; i != end; ++i) @@ -161,6 +165,115 @@ namespace DB } } + template + static PaddedPODArray extractIndexesImpl(ColumnPtr column, size_t start, size_t end) + { + const PaddedPODArray & data = checkAndGetColumn>(*column)->getData(); + PaddedPODArray result; + result.reserve(end - start); + std::transform(data.begin() + start, data.begin() + end, std::back_inserter(result), [](T value) { return Int64(value); }); + return result; + } + + static PaddedPODArray extractIndexesImpl(ColumnPtr column, size_t start, size_t end) + { + switch (column->getDataType()) + { + case TypeIndex::UInt8: + return extractIndexesImpl(column, start, end); + case TypeIndex::UInt16: + return extractIndexesImpl(column, start, end); + case TypeIndex::UInt32: + return extractIndexesImpl(column, start, end); + case TypeIndex::UInt64: + return extractIndexesImpl(column, start, end); + default: + throw Exception("Indexes column must be ColumnUInt, got " + column->getName(), + ErrorCodes::LOGICAL_ERROR); + } + } + + template + static void fillArrowArrayWithLowCardinalityColumnDataImpl( + const String & column_name, + ColumnPtr & column, + const std::shared_ptr & column_type, + const PaddedPODArray * null_bytemap, + arrow::ArrayBuilder * array_builder, + String format_name, + size_t start, + size_t end, + std::unordered_map> & dictionary_values) + { + const auto * column_lc = typeid_cast(column.get()); + arrow::DictionaryBuilder * builder = typeid_cast *>(array_builder); + auto & dict_values = dictionary_values[column_name]; + + /// Convert dictionary from LowCardinality to Arrow dictionary only once and then reuse it. + if (!dict_values) + { + auto value_type = typeid_cast(builder->type().get())->value_type(); + std::unique_ptr values_builder; + arrow::MemoryPool* pool = arrow::default_memory_pool(); + arrow::Status status = MakeBuilder(pool, value_type, &values_builder); + checkStatus(status, column->getName(), format_name); + + auto dict_column = column_lc->getDictionary().getNestedColumn(); + const auto & dict_type = typeid_cast(column_type.get())->getDictionaryType(); + fillArrowArray(column_name, dict_column, dict_type, nullptr, values_builder.get(), format_name, 0, dict_column->size(), dictionary_values); + status = values_builder->Finish(&dict_values); + checkStatus(status, column->getName(), format_name); + } + + arrow::Status status = builder->InsertMemoValues(*dict_values); + checkStatus(status, column->getName(), format_name); + + /// AppendIndices in DictionaryBuilder works only with int64_t data, so we cannot use + /// fillArrowArray here and should copy all indexes to int64_t container. + auto indexes = extractIndexesImpl(column_lc->getIndexesPtr(), start, end); + const uint8_t * arrow_null_bytemap_raw_ptr = nullptr; + PaddedPODArray arrow_null_bytemap; + if (null_bytemap) + { + /// Invert values since Arrow interprets 1 as a non-null value, while CH as a null + arrow_null_bytemap.reserve(end - start); + for (size_t i = start; i < end; ++i) + arrow_null_bytemap.emplace_back(!(*null_bytemap)[i]); + + arrow_null_bytemap_raw_ptr = arrow_null_bytemap.data(); + } + + status = builder->AppendIndices(indexes.data(), indexes.size(), arrow_null_bytemap_raw_ptr); + checkStatus(status, column->getName(), format_name); + } + + + static void fillArrowArrayWithLowCardinalityColumnData( + const String & column_name, + ColumnPtr & column, + const std::shared_ptr & column_type, + const PaddedPODArray * null_bytemap, + arrow::ArrayBuilder * array_builder, + String format_name, + size_t start, + size_t end, + std::unordered_map> & dictionary_values) + { + auto * dict_type = typeid_cast(array_builder->type().get()); + auto value_type = dict_type->value_type(); + +#define DISPATCH(ARROW_TYPE_ID, ARROW_TYPE) \ + if (arrow::Type::ARROW_TYPE_ID == value_type->id()) \ + { \ + fillArrowArrayWithLowCardinalityColumnDataImpl(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); \ + return; \ + } + + FOR_ARROW_TYPES(DISPATCH) +#undef DISPATCH + + } + template static void fillArrowArrayWithStringColumnData( ColumnPtr write_column, @@ -185,7 +298,6 @@ namespace DB StringRef string_ref = internal_column.getDataAt(string_i); status = builder.Append(string_ref.data, string_ref.size); } - checkStatus(status, write_column->getName(), format_name); } } @@ -248,7 +360,8 @@ namespace DB arrow::ArrayBuilder * array_builder, String format_name, size_t start, - size_t end) + size_t end, + std::unordered_map> & dictionary_values) { const String column_type_name = column_type->getFamilyName(); @@ -259,7 +372,7 @@ namespace DB DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); const ColumnPtr & null_column = column_nullable->getNullMapColumnPtr(); const PaddedPODArray & bytemap = assert_cast &>(*null_column).getData(); - fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end); + fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end, dictionary_values); } else if ("String" == column_type_name) { @@ -279,11 +392,15 @@ namespace DB } else if ("Array" == column_type_name) { - fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end); + fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); } else if ("Tuple" == column_type_name) { - fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end); + fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); + } + else if ("LowCardinality" == column_type_name) + { + fillArrowArrayWithLowCardinalityColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); } else if (isDecimal(column_type)) { @@ -346,12 +463,33 @@ namespace DB checkStatus(status, write_column->getName(), format_name); } - static std::shared_ptr getArrowType(DataTypePtr column_type, const std::string & column_name, const std::string & format_name, bool * is_column_nullable) + static std::shared_ptr getArrowTypeForLowCardinalityIndexes(ColumnPtr indexes_column) + { + /// Arrow docs recommend preferring signed integers over unsigned integers for representing dictionary indices. + /// https://arrow.apache.org/docs/format/Columnar.html#dictionary-encoded-layout + switch (indexes_column->getDataType()) + { + case TypeIndex::UInt8: + return arrow::int8(); + case TypeIndex::UInt16: + return arrow::int16(); + case TypeIndex::UInt32: + return arrow::int32(); + case TypeIndex::UInt64: + return arrow::int64(); + default: + throw Exception("Indexes column for getUniqueIndex must be ColumnUInt, got " + indexes_column->getName(), + ErrorCodes::LOGICAL_ERROR); + } + } + + static std::shared_ptr getArrowType(DataTypePtr column_type, ColumnPtr column, const std::string & column_name, const std::string & format_name, bool * is_column_nullable) { if (column_type->isNullable()) { DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); - auto arrow_type = getArrowType(nested_type, column_name, format_name, is_column_nullable); + ColumnPtr nested_column = checkAndGetColumn(*column)->getNestedColumnPtr(); + auto arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable); *is_column_nullable = true; return arrow_type; } @@ -381,23 +519,36 @@ namespace DB if (isArray(column_type)) { auto nested_type = typeid_cast(column_type.get())->getNestedType(); - auto nested_arrow_type = getArrowType(nested_type, column_name, format_name, is_column_nullable); + auto nested_column = checkAndGetColumn(*column)->getDataPtr(); + auto nested_arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable); return arrow::list(nested_arrow_type); } if (isTuple(column_type)) { const auto & nested_types = typeid_cast(column_type.get())->getElements(); + const auto * tuple_column = checkAndGetColumn(*column); std::vector> nested_fields; for (size_t i = 0; i != nested_types.size(); ++i) { String name = column_name + "." + std::to_string(i); - auto nested_arrow_type = getArrowType(nested_types[i], name, format_name, is_column_nullable); + auto nested_arrow_type = getArrowType(nested_types[i], tuple_column->getColumnPtr(i), name, format_name, is_column_nullable); nested_fields.push_back(std::make_shared(name, nested_arrow_type, *is_column_nullable)); } return arrow::struct_(std::move(nested_fields)); } + if (column_type->lowCardinality()) + { + auto nested_type = typeid_cast(column_type.get())->getDictionaryType(); + const auto * lc_column = checkAndGetColumn(*column); + auto nested_column = lc_column->getDictionaryPtr(); + auto indexes_column = lc_column->getIndexesPtr(); + return arrow::dictionary( + getArrowTypeForLowCardinalityIndexes(indexes_column), + getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable)); + } + const std::string type_name = column_type->getFamilyName(); if (const auto * arrow_type_it = std::find_if( internal_type_to_arrow_type.begin(), @@ -418,7 +569,8 @@ namespace DB const Block & header, const Chunk & chunk, size_t columns_num, - String format_name) + String format_name, + bool low_cardinality_as_dictionary) { /// For arrow::Schema and arrow::Table creation std::vector> arrow_fields; @@ -430,11 +582,16 @@ namespace DB { // TODO: constructed every iteration ColumnWithTypeAndName column = header.safeGetByPosition(column_i); - column.column = recursiveRemoveLowCardinality(chunk.getColumns()[column_i]); - column.type = recursiveRemoveLowCardinality(column.type); + column.column = chunk.getColumns()[column_i]; + + if (!low_cardinality_as_dictionary) + { + column.column = recursiveRemoveLowCardinality(column.column); + column.type = recursiveRemoveLowCardinality(column.type); + } bool is_column_nullable = false; - auto arrow_type = getArrowType(column.type, column.name, format_name, &is_column_nullable); + auto arrow_type = getArrowType(column.type, column.column, column.name, format_name, &is_column_nullable); arrow_fields.emplace_back(std::make_shared(column.name, arrow_type, is_column_nullable)); arrow::MemoryPool* pool = arrow::default_memory_pool(); @@ -442,7 +599,7 @@ namespace DB arrow::Status status = MakeBuilder(pool, arrow_fields[column_i]->type(), &array_builder); checkStatus(status, column.column->getName(), format_name); - fillArrowArray(column.name, column.column, column.type, nullptr, array_builder.get(), format_name, 0, column.column->size()); + fillArrowArray(column.name, column.column, column.type, nullptr, array_builder.get(), format_name, 0, column.column->size(), dictionary_values); std::shared_ptr arrow_array; status = array_builder->Finish(&arrow_array); diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h index de594389c25..9df70b5bceb 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h @@ -26,10 +26,27 @@ private: M(Float32, arrow::FloatBuilder) \ M(Float64, arrow::DoubleBuilder) +#define FOR_ARROW_TYPES(M) \ + M(UINT8, arrow::UInt8Type) \ + M(INT8, arrow::Int8Type) \ + M(UINT16, arrow::UInt16Type) \ + M(INT16, arrow::Int16Type) \ + M(UINT32, arrow::UInt32Type) \ + M(INT32, arrow::Int32Type) \ + M(UINT64, arrow::UInt64Type) \ + M(INT64, arrow::Int64Type) \ + M(FLOAT, arrow::FloatType) \ + M(DOUBLE, arrow::DoubleType) \ + M(STRING, arrow::StringType) + + /// Map {column name : arrow dictionary}. + /// To avoid converting dictionary from LowCardinality to Arrow + /// Dictionary every chunk we save it and reuse. + std::unordered_map> dictionary_values; public: - static void chChunkToArrowTable(std::shared_ptr & res, const Block & header, const Chunk & chunk, - size_t columns_num, String format_name); + void chChunkToArrowTable(std::shared_ptr & res, const Block & header, const Chunk & chunk, + size_t columns_num, String format_name, bool low_cardinality_as_dictionary = false); }; } #endif diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index d16d0870474..7b1c2af1a52 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -54,7 +54,7 @@ Chunk ORCBlockInputFormat::generate() ++stripe_current; - ArrowColumnToCHColumn::arrowTableToCHChunk(res, *table_result, header, "ORC"); + arrow_column_to_ch_column.arrowTableToCHChunk(res, *table_result, header, "ORC"); return res; } @@ -72,7 +72,15 @@ static size_t countIndicesForType(std::shared_ptr type) if (type->id() == arrow::Type::LIST) return countIndicesForType(static_cast(type.get())->value_type()) + 1; - return 1; + int indices = 1; + if (type->id() == arrow::Type::STRUCT) + { + auto * struct_type = static_cast(type.get()); + for (int i = 0; i != struct_type->num_fields(); ++i) + indices += countIndicesForType(struct_type->field(i)->type()); + } + + return indices; } void ORCBlockInputFormat::prepareReader() @@ -89,8 +97,8 @@ void ORCBlockInputFormat::prepareReader() int index = 1; for (int i = 0; i < schema->num_fields(); ++i) { - /// LIST type require 2 indices, so we should recursively - /// count the number of indices we need for this type. + /// LIST type require 2 indices, STRUCT - the number of elements + 1, + /// so we should recursively count the number of indices we need for this type. int indexes_count = countIndicesForType(schema->field(i)->type()); if (getPort().getHeader().has(schema->field(i)->name())) { diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index 0c78290f3cc..5a6cfd1364a 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -3,6 +3,7 @@ #if USE_ORC #include +#include namespace arrow::adapters::orc { class ORCFileReader; } @@ -26,6 +27,8 @@ private: std::unique_ptr file_reader; + ArrowColumnToCHColumn arrow_column_to_ch_column; + int stripe_total = 0; int stripe_current = 0; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 0223e5219c0..1d7feadc1e7 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -54,7 +54,7 @@ Chunk ParquetBlockInputFormat::generate() ++row_group_current; - ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, header, "Parquet"); + arrow_column_to_ch_column.arrowTableToCHChunk(res, table, header, "Parquet"); return res; } diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index 0841e82d4d0..b27bafe04bf 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -4,6 +4,7 @@ #if USE_PARQUET #include +#include namespace parquet::arrow { class FileReader; } @@ -32,6 +33,7 @@ private: int row_group_total = 0; // indices of columns to read from Parquet file std::vector column_indices; + ArrowColumnToCHColumn arrow_column_to_ch_column; int row_group_current = 0; }; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 78b6a9c53a4..25eade4bafa 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -36,7 +36,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet"); + ch_column_to_arrow_column.chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet"); if (!file_writer) { diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index 11d746a0a6d..deb011e0274 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -4,6 +4,7 @@ #if USE_PARQUET # include # include +# include namespace arrow { @@ -36,6 +37,7 @@ private: const FormatSettings format_settings; std::unique_ptr file_writer; + CHColumnToArrowColumn ch_column_to_arrow_column; }; } diff --git a/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference b/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference new file mode 100644 index 00000000000..b144130918f --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference @@ -0,0 +1,4 @@ +1 ['a','b','c'] ('z','6') +2 ['d','e'] ('x','9') +1 ['a','b','c'] ('z','6') +2 ['d','e'] ('x','9') diff --git a/tests/queries/0_stateless/01273_arrow_dictionaries_load.sh b/tests/queries/0_stateless/01273_arrow_dictionaries_load.sh new file mode 100755 index 00000000000..38e6c2c1b01 --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_dictionaries_load.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_dicts" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_dicts (a LowCardinality(String), b Array(LowCardinality(String)), c Tuple(LowCardinality(String), LowCardinality(String))) ENGINE=Memory()" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_dicts VALUES ('1', ['a', 'b', 'c'], ('z', '6')), ('2', ['d', 'e'], ('x', '9'))" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts FORMAT Arrow SETTINGS output_format_arrow_low_cardinality_as_dictionary=1" > "${CLICKHOUSE_TMP}"/dicts.arrow + +cat "${CLICKHOUSE_TMP}"/dicts.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_dicts FORMAT Arrow" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_dicts" + +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_dicts (a LowCardinality(String)) ENGINE=Memory()" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_dicts SELECT toString(number % 500) from numbers(10000000)" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts FORMAT Arrow SETTINGS output_format_arrow_low_cardinality_as_dictionary=1" > "${CLICKHOUSE_TMP}"/dicts.arrow + +cat "${CLICKHOUSE_TMP}"/dicts.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_dicts FORMAT Arrow" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_dicts" + From a4decd08484d07928c668dfdfb8bb08eb4f0ba06 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 27 May 2021 22:01:06 +0300 Subject: [PATCH 200/491] Support Map type, fix and add tests --- src/Core/Settings.h | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 41 +++++++--- .../Formats/Impl/CHColumnToArrowColumn.cpp | 77 ++++++++++++------- .../Formats/Impl/ORCBlockInputFormat.cpp | 14 +++- .../Formats/Impl/ORCBlockOutputFormat.cpp | 51 ++++++++++-- .../Formats/Impl/ParquetBlockInputFormat.cpp | 6 ++ .../00900_long_parquet_load.reference | 28 +++++-- .../0_stateless/00900_long_parquet_load.sh | 5 +- .../00900_orc_arrow_parquet_maps.reference | 6 ++ .../00900_orc_arrow_parquet_maps.sh | 28 +++++++ ... 00900_orc_arrow_parquet_tuples.reference} | 4 + .../00900_orc_arrow_parquet_tuples.sh | 24 ++++++ .../queries/0_stateless/00900_orc_map_load.sh | 20 +++++ .../0_stateless/00900_orc_tuples_load.sh | 17 ---- .../00900_parquet_tuples_load.reference | 2 - .../0_stateless/00900_parquet_tuples_load.sh | 17 ---- .../01273_arrow_tuples_load.reference | 2 - .../0_stateless/01273_arrow_tuples_load.sh | 17 ---- .../nested_maps.snappy.parquet.columns | 2 +- .../nonnullable.impala.parquet.columns | 2 +- .../nullable.impala.parquet.columns | 2 +- .../data_parquet/nulls.snappy.parquet.columns | 2 +- 22 files changed, 256 insertions(+), 113 deletions(-) create mode 100644 tests/queries/0_stateless/00900_orc_arrow_parquet_maps.reference create mode 100755 tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh rename tests/queries/0_stateless/{00900_orc_tuples_load.reference => 00900_orc_arrow_parquet_tuples.reference} (63%) create mode 100755 tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh create mode 100644 tests/queries/0_stateless/00900_orc_map_load.sh delete mode 100755 tests/queries/0_stateless/00900_orc_tuples_load.sh delete mode 100644 tests/queries/0_stateless/00900_parquet_tuples_load.reference delete mode 100755 tests/queries/0_stateless/00900_parquet_tuples_load.sh delete mode 100644 tests/queries/0_stateless/01273_arrow_tuples_load.reference delete mode 100755 tests/queries/0_stateless/01273_arrow_tuples_load.sh diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4815f060140..dae186e70ba 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -117,7 +117,7 @@ class IColumn; M(Bool, skip_unavailable_shards, false, "If 1, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ - M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards. If 2 - same as 1 but also apply ORDER BY and LIMIT stages", 0) \ + M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 23987067474..8f76f5dca8a 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include @@ -293,14 +295,15 @@ namespace DB { if (internal_column.isNullable()) { - ColumnNullable & column_nullable = typeid_cast(internal_column); + ColumnNullable & column_nullable = assert_cast(internal_column); readColumnFromArrowColumn(arrow_column, column_nullable.getNestedColumn(), column_name, format_name, true, dictionary_values); fillByteMapFromArrowColumn(arrow_column, column_nullable.getNullMapColumn()); return; } - // TODO: check if a column is const? - if (!is_nullable && !checkColumn(internal_column) && arrow_column->null_count()) + /// TODO: check if a column is const? + if (!is_nullable && arrow_column->null_count() && arrow_column->type()->id() != arrow::Type::LIST + && arrow_column->type()->id() != arrow::Type::MAP && arrow_column->type()->id() != arrow::Type::STRUCT) { throw Exception { @@ -332,6 +335,7 @@ namespace DB //fillColumnWithNumericData>(arrow_column, read_column); // Have problems with trash values under NULL, but faster fillColumnWithDecimalData(arrow_column, internal_column /*, internal_nested_type*/); break; + case arrow::Type::MAP: [[fallthrough]]; case arrow::Type::LIST: { arrow::ArrayVector array_vector; @@ -344,14 +348,17 @@ namespace DB } auto arrow_nested_column = std::make_shared(array_vector); - ColumnArray & column_array = typeid_cast(internal_column); + ColumnArray & column_array = arrow_column->type()->id() == arrow::Type::MAP + ? assert_cast(internal_column).getNestedColumn() + : assert_cast(internal_column); + readColumnFromArrowColumn(arrow_nested_column, column_array.getData(), column_name, format_name, false, dictionary_values); fillOffsetsFromArrowListColumn(arrow_column, column_array.getOffsetsColumn()); break; } case arrow::Type::STRUCT: { - ColumnTuple & column_tuple = typeid_cast(internal_column); + ColumnTuple & column_tuple = assert_cast(internal_column); int fields_count = column_tuple.tupleSize(); std::vector nested_arrow_columns(fields_count); for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) @@ -370,7 +377,7 @@ namespace DB } case arrow::Type::DICTIONARY: { - ColumnLowCardinality & column_lc = typeid_cast(internal_column); + ColumnLowCardinality & column_lc = assert_cast(internal_column); auto & dict_values = dictionary_values[column_name]; /// Load dictionary values only once and reuse it. if (!dict_values) @@ -430,7 +437,7 @@ namespace DB { if (column_type->isNullable()) { - DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); + DataTypePtr nested_type = assert_cast(column_type.get())->getNestedType(); return makeNullable(getInternalType(arrow_type, nested_type, column_name, format_name)); } @@ -447,7 +454,7 @@ namespace DB const DataTypeArray * array_type = typeid_cast(column_type.get()); if (!array_type) - throw Exception{"Cannot convert arrow LIST type to a not Array ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + throw Exception{"Cannot convert arrow LIST type to a not Array/Map ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; return std::make_shared(getInternalType(list_nested_type, array_type->getNestedType(), column_name, format_name)); } @@ -478,7 +485,23 @@ namespace DB if (arrow_type->id() == arrow::Type::DICTIONARY) { const auto * arrow_dict_type = static_cast(arrow_type.get()); - return std::make_shared(getInternalType(arrow_dict_type->value_type(), column_type, column_name, format_name)); + const auto * lc_type = typeid_cast(column_type.get()); + /// We allow to insert arrow dictionary into a non-LowCardinality column. + const auto & dict_type = lc_type ? lc_type->getDictionaryType() : column_type; + return std::make_shared(getInternalType(arrow_dict_type->value_type(), dict_type, column_name, format_name)); + } + + if (arrow_type->id() == arrow::Type::MAP) + { + const auto * arrow_map_type = typeid_cast(arrow_type.get()); + const auto * map_type = typeid_cast(column_type.get()); + if (!map_type) + throw Exception{"Cannot convert arrow MAP type to a not Map ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + + return std::make_shared( + getInternalType(arrow_map_type->key_type(), map_type->getKeyType(), column_name, format_name), + getInternalType(arrow_map_type->item_type(), map_type->getValueType(), column_name, format_name) + ); } if (const auto * internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(), diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 97a5326a421..7041193e09a 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +109,7 @@ namespace DB size_t end, std::unordered_map> & dictionary_values); + template static void fillArrowArrayWithArrayColumnData( const String & column_name, ColumnPtr & column, @@ -118,12 +121,12 @@ namespace DB size_t end, std::unordered_map> & dictionary_values) { - const auto * column_array = typeid_cast(column.get()); + const auto * column_array = assert_cast(column.get()); ColumnPtr nested_column = column_array->getDataPtr(); - DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); + DataTypePtr nested_type = assert_cast(column_type.get())->getNestedType(); const auto & offsets = column_array->getOffsets(); - arrow::ListBuilder & builder = assert_cast(*array_builder); + Builder & builder = assert_cast(*array_builder); arrow::ArrayBuilder * value_builder = builder.value_builder(); arrow::Status components_status; @@ -147,8 +150,8 @@ namespace DB size_t end, std::unordered_map> & dictionary_values) { - const auto * column_tuple = typeid_cast(column.get()); - const auto & nested_types = typeid_cast(column_type.get())->getElements(); + const auto * column_tuple = assert_cast(column.get()); + const auto & nested_types = assert_cast(column_type.get())->getElements(); arrow::StructBuilder & builder = assert_cast(*array_builder); @@ -168,7 +171,7 @@ namespace DB template static PaddedPODArray extractIndexesImpl(ColumnPtr column, size_t start, size_t end) { - const PaddedPODArray & data = checkAndGetColumn>(*column)->getData(); + const PaddedPODArray & data = assert_cast *>(column.get())->getData(); PaddedPODArray result; result.reserve(end - start); std::transform(data.begin() + start, data.begin() + end, std::back_inserter(result), [](T value) { return Int64(value); }); @@ -205,21 +208,21 @@ namespace DB size_t end, std::unordered_map> & dictionary_values) { - const auto * column_lc = typeid_cast(column.get()); - arrow::DictionaryBuilder * builder = typeid_cast *>(array_builder); + const auto * column_lc = assert_cast(column.get()); + arrow::DictionaryBuilder * builder = assert_cast *>(array_builder); auto & dict_values = dictionary_values[column_name]; /// Convert dictionary from LowCardinality to Arrow dictionary only once and then reuse it. if (!dict_values) { - auto value_type = typeid_cast(builder->type().get())->value_type(); + const auto & value_type = assert_cast(builder->type().get())->value_type(); std::unique_ptr values_builder; arrow::MemoryPool* pool = arrow::default_memory_pool(); arrow::Status status = MakeBuilder(pool, value_type, &values_builder); checkStatus(status, column->getName(), format_name); auto dict_column = column_lc->getDictionary().getNestedColumn(); - const auto & dict_type = typeid_cast(column_type.get())->getDictionaryType(); + const auto & dict_type = assert_cast(column_type.get())->getDictionaryType(); fillArrowArray(column_name, dict_column, dict_type, nullptr, values_builder.get(), format_name, 0, dict_column->size(), dictionary_values); status = values_builder->Finish(&dict_values); checkStatus(status, column->getName(), format_name); @@ -259,8 +262,7 @@ namespace DB size_t end, std::unordered_map> & dictionary_values) { - auto * dict_type = typeid_cast(array_builder->type().get()); - auto value_type = dict_type->value_type(); + auto value_type = assert_cast(array_builder->type().get())->value_type(); #define DISPATCH(ARROW_TYPE_ID, ARROW_TYPE) \ if (arrow::Type::ARROW_TYPE_ID == value_type->id()) \ @@ -367,9 +369,9 @@ namespace DB if ("Nullable" == column_type_name) { - const ColumnNullable * column_nullable = checkAndGetColumn(column.get()); + const ColumnNullable * column_nullable = assert_cast(column.get()); ColumnPtr nested_column = column_nullable->getNestedColumnPtr(); - DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); + DataTypePtr nested_type = assert_cast(column_type.get())->getNestedType(); const ColumnPtr & null_column = column_nullable->getNullMapColumnPtr(); const PaddedPODArray & bytemap = assert_cast &>(*null_column).getData(); fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end, dictionary_values); @@ -392,7 +394,7 @@ namespace DB } else if ("Array" == column_type_name) { - fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); + fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); } else if ("Tuple" == column_type_name) { @@ -402,6 +404,12 @@ namespace DB { fillArrowArrayWithLowCardinalityColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); } + else if ("Map" == column_type_name) + { + ColumnPtr column_array = assert_cast(column.get())->getNestedColumnPtr(); + DataTypePtr array_type = assert_cast(column_type.get())->getNestedType(); + fillArrowArrayWithArrayColumnData(column_name, column_array, array_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); + } else if (isDecimal(column_type)) { auto fill_decimal = [&](const auto & types) -> bool @@ -446,7 +454,7 @@ namespace DB size_t start, size_t end) { - const auto & column = static_cast(*write_column); + const auto & column = assert_cast(*write_column); arrow::DecimalBuilder & builder = assert_cast(*array_builder); arrow::Status status; @@ -487,8 +495,8 @@ namespace DB { if (column_type->isNullable()) { - DataTypePtr nested_type = typeid_cast(column_type.get())->getNestedType(); - ColumnPtr nested_column = checkAndGetColumn(*column)->getNestedColumnPtr(); + DataTypePtr nested_type = assert_cast(column_type.get())->getNestedType(); + ColumnPtr nested_column = assert_cast(column.get())->getNestedColumnPtr(); auto arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable); *is_column_nullable = true; return arrow_type; @@ -506,7 +514,7 @@ namespace DB || std::is_same_v> || std::is_same_v>) { - const auto & decimal_type = static_cast(column_type.get()); + const auto & decimal_type = assert_cast(column_type.get()); arrow_type = arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale()); } @@ -518,16 +526,16 @@ namespace DB if (isArray(column_type)) { - auto nested_type = typeid_cast(column_type.get())->getNestedType(); - auto nested_column = checkAndGetColumn(*column)->getDataPtr(); + auto nested_type = assert_cast(column_type.get())->getNestedType(); + auto nested_column = assert_cast(column.get())->getDataPtr(); auto nested_arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable); return arrow::list(nested_arrow_type); } if (isTuple(column_type)) { - const auto & nested_types = typeid_cast(column_type.get())->getElements(); - const auto * tuple_column = checkAndGetColumn(*column); + const auto & nested_types = assert_cast(column_type.get())->getElements(); + const auto * tuple_column = assert_cast(column.get()); std::vector> nested_fields; for (size_t i = 0; i != nested_types.size(); ++i) { @@ -540,15 +548,28 @@ namespace DB if (column_type->lowCardinality()) { - auto nested_type = typeid_cast(column_type.get())->getDictionaryType(); - const auto * lc_column = checkAndGetColumn(*column); - auto nested_column = lc_column->getDictionaryPtr(); - auto indexes_column = lc_column->getIndexesPtr(); + auto nested_type = assert_cast(column_type.get())->getDictionaryType(); + const auto * lc_column = assert_cast(column.get()); + ColumnPtr nested_column = lc_column->getDictionaryPtr(); + ColumnPtr indexes_column = lc_column->getIndexesPtr(); return arrow::dictionary( getArrowTypeForLowCardinalityIndexes(indexes_column), getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable)); } + if (isMap(column_type)) + { + const auto * map_type = assert_cast(column_type.get()); + const auto & key_type = map_type->getKeyType(); + const auto & val_type = map_type->getValueType(); + + const auto & columns = assert_cast(column.get())->getNestedData().getColumns(); + return arrow::map( + getArrowType(key_type, columns[0], column_name, format_name, is_column_nullable), + getArrowType(val_type, columns[1], column_name, format_name, is_column_nullable) + ); + } + const std::string type_name = column_type->getFamilyName(); if (const auto * arrow_type_it = std::find_if( internal_type_to_arrow_type.begin(), @@ -559,7 +580,7 @@ namespace DB return arrow_type_it->second; } - throw Exception{"The type \"" + column_name + "\" of a column \"" + column_name + "\"" + throw Exception{"The type \"" + column_type->getName() + "\" of a column \"" + column_name + "\"" " is not supported for conversion into a " + format_name + " data format", ErrorCodes::UNKNOWN_TYPE}; } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 7b1c2af1a52..cb503def801 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -72,15 +72,25 @@ static size_t countIndicesForType(std::shared_ptr type) if (type->id() == arrow::Type::LIST) return countIndicesForType(static_cast(type.get())->value_type()) + 1; - int indices = 1; if (type->id() == arrow::Type::STRUCT) { + int indices = 1; auto * struct_type = static_cast(type.get()); for (int i = 0; i != struct_type->num_fields(); ++i) indices += countIndicesForType(struct_type->field(i)->type()); + return indices; } - return indices; + if (type->id() == arrow::Type::MAP) + { + int indices = 0; + auto * map_type = static_cast(type.get()); + indices += countIndicesForType(map_type->key_type()); + indices += countIndicesForType(map_type->item_type()); + return indices; + } + + return 1; } void ORCBlockInputFormat::prepareReader() diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index a560701944d..a5143792e7d 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include namespace DB { @@ -102,27 +104,27 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t } case TypeIndex::Array: { - const auto * array_type = typeid_cast(type.get()); + const auto * array_type = assert_cast(type.get()); return orc::createListType(getORCType(array_type->getNestedType(), column_name)); } case TypeIndex::Decimal32: { - const auto * decimal_type = typeid_cast *>(type.get()); + const auto * decimal_type = assert_cast *>(type.get()); return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); } case TypeIndex::Decimal64: { - const auto * decimal_type = typeid_cast *>(type.get()); + const auto * decimal_type = assert_cast *>(type.get()); return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); } case TypeIndex::Decimal128: { - const auto * decimal_type = typeid_cast *>(type.get()); + const auto * decimal_type = assert_cast *>(type.get()); return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); } case TypeIndex::Tuple: { - const auto * tuple_type = typeid_cast(type.get()); + const auto * tuple_type = assert_cast(type.get()); const auto & nested_types = tuple_type->getElements(); auto struct_type = orc::createStructType(); for (size_t i = 0; i < nested_types.size(); ++i) @@ -132,6 +134,14 @@ ORC_UNIQUE_PTR ORCBlockOutputFormat::getORCType(const DataTypePtr & t } return struct_type; } + case TypeIndex::Map: + { + const auto * map_type = assert_cast(type.get()); + return orc::createMapType( + getORCType(map_type->getKeyType(), column_name), + getORCType(map_type->getValueType(), column_name) + ); + } default: { throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN); @@ -174,7 +184,7 @@ void ORCBlockOutputFormat::writeDecimals( { DecimalVectorBatch & decimal_orc_column = dynamic_cast(orc_column); const auto & decimal_column = assert_cast &>(column); - const auto * decimal_type = typeid_cast *>(type.get()); + const auto * decimal_type = assert_cast *>(type.get()); decimal_orc_column.precision = decimal_type->getPrecision(); decimal_orc_column.scale = decimal_type->getScale(); decimal_orc_column.resize(decimal_column.size()); @@ -400,13 +410,40 @@ void ORCBlockOutputFormat::writeColumn( { orc::StructVectorBatch & struct_orc_column = dynamic_cast(orc_column); const auto & tuple_column = assert_cast(column); - auto nested_types = typeid_cast(type.get())->getElements(); + auto nested_types = assert_cast(type.get())->getElements(); for (size_t i = 0; i != tuple_column.size(); ++i) struct_orc_column.notNull[i] = 1; for (size_t i = 0; i != tuple_column.tupleSize(); ++i) writeColumn(*struct_orc_column.fields[i], tuple_column.getColumn(i), nested_types[i], null_bytemap); break; + } + case TypeIndex::Map: + { + orc::MapVectorBatch & map_orc_column = dynamic_cast(orc_column); + const auto & list_column = assert_cast(column).getNestedColumn(); + const auto & map_type = assert_cast(*type); + const ColumnArray::Offsets & offsets = list_column.getOffsets(); + map_orc_column.resize(list_column.size()); + /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. + map_orc_column.offsets[0] = 0; + for (size_t i = 0; i != list_column.size(); ++i) + { + map_orc_column.offsets[i + 1] = offsets[i]; + map_orc_column.notNull[i] = 1; + } + const auto nested_columns = assert_cast(list_column.getDataPtr().get())->getColumns(); + + orc::ColumnVectorBatch & keys_orc_column = *map_orc_column.keys; + auto key_type = map_type.getKeyType(); + writeColumn(keys_orc_column, *nested_columns[0], key_type, null_bytemap); + + orc::ColumnVectorBatch & values_orc_column = *map_orc_column.elements; + auto value_type = map_type.getValueType(); + writeColumn(values_orc_column, *nested_columns[1], value_type, null_bytemap); + + map_orc_column.numElements = list_column.size(); + break; } default: throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 1d7feadc1e7..bde39446dbb 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -79,6 +79,12 @@ static size_t countIndicesForType(std::shared_ptr type) for (int i = 0; i != struct_type->num_fields(); ++i) indices += countIndicesForType(struct_type->field(i)->type()); } + else if (type->id() == arrow::Type::MAP) + { + auto * map_type = static_cast(type.get()); + indices += countIndicesForType(map_type->key_type()); + indices += countIndicesForType(map_type->item_type()); + } else indices = 1; diff --git a/tests/queries/0_stateless/00900_long_parquet_load.reference b/tests/queries/0_stateless/00900_long_parquet_load.reference index 4bd699f40fe..621bca2ec0e 100644 --- a/tests/queries/0_stateless/00900_long_parquet_load.reference +++ b/tests/queries/0_stateless/00900_long_parquet_load.reference @@ -298,24 +298,40 @@ Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Not y [[['a','b'],['c','d']],[[],['e']]] 1 [[['a','b'],['c','d'],['e']],[[],['f']]] 1 === Try load data from nested_maps.snappy.parquet -Code: 70. DB::Ex---tion: The type "map" of an input column "a" is not supported for conversion from a Parquet data format: data for INSERT was parsed from stdin - +{'a':{1:1,2:0}} 1 1 +{'b':{1:1}} 1 1 +{'c':{}} 1 1 +{'d':{}} 1 1 +{'e':{1:1}} 1 1 +{'f':{3:1,4:0,5:1}} 1 1 === Try load data from non_hadoop_lz4_compressed.parquet 1593604800 abc 42 1593604800 def 7.7 1593604801 abc 42.125 1593604801 def 7.7 === Try load data from nonnullable.impala.parquet -../contrib/arrow/cpp/src/arrow/array/array_nested.cc:192: Check failed: (self->list_type_->value_type()->id()) == (data->child_data[0]->type->id()) +8 [-1] [[-1,-2],[]] {'k1':-1} [{},{'k1':1},{},{}] (-1,[-1],([[(-1)]]),{}) === Try load data from nullable.impala.parquet -../contrib/arrow/cpp/src/arrow/array/array_nested.cc:192: Check failed: (self->list_type_->value_type()->id()) == (data->child_data[0]->type->id()) +1 [1,2,3] [[1,2],[3,4]] {'k1':1,'k2':100} [{'k1':1}] (1,[1],([[(10),(-10)],[(11)]]),{'foo':(([1.1]))}) +2 [NULL,1,2,NULL,3,NULL] [[NULL,1,2,NULL],[3,NULL,4],[],[]] {'k1':2,'k2':NULL} [{'k3':NULL,'k1':1},{},{}] (NULL,[NULL],([[(NULL),(10),(NULL),(-10),(NULL)],[(11),(NULL)],[],[]]),{'g1':(([2.2,NULL])),'g2':(([])),'g3':(([])),'g4':(([])),'g5':(([]))}) +3 [] [[]] {} [{},{}] (NULL,[],([]),{}) +4 [] [] {} [] (NULL,[],([]),{}) +5 [] [] {} [] (NULL,[],([]),{'foo':(([2.2,3.3]))}) +6 [] [] {} [] (NULL,[],([]),{}) +7 [] [[],[5,6]] {'k1':NULL,'k3':NULL} [] (7,[2,3,NULL],([[],[(NULL)],[]]),{}) === Try load data from nullable_list.parquet [1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] [NULL] [NULL] [NULL] [] [] [] === Try load data from nulls.snappy.parquet -Code: 70. DB::Ex---tion: The type "struct" of an input column "b_struct" is not supported for conversion from a Parquet data format: data for INSERT was parsed from stdin - +(NULL) +(NULL) +(NULL) +(NULL) +(NULL) +(NULL) +(NULL) +(NULL) === Try load data from single_nan.parquet \N === Try load data from userdata1.parquet diff --git a/tests/queries/0_stateless/00900_long_parquet_load.sh b/tests/queries/0_stateless/00900_long_parquet_load.sh index 52213f066e1..1cfba22587a 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load.sh @@ -55,7 +55,10 @@ for NAME in $(find "$DATA_DIR"/*.parquet -print0 | xargs -0 -n 1 basename | LC_A COLUMNS=$(cat "$COLUMNS_FILE") || continue ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load" - ${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load ($COLUMNS) ENGINE = Memory" + $CLICKHOUSE_CLIENT --multiquery <&1 | sed 's/Exception/Ex---tion/' diff --git a/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.reference b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.reference new file mode 100644 index 00000000000..d96eb672258 --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.reference @@ -0,0 +1,6 @@ +Arrow +{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}] +Parquet +{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}] +ORC +{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}] diff --git a/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh new file mode 100755 index 00000000000..9330a5924a9 --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS maps" +${CLICKHOUSE_CLIENT} --multiquery < "${CLICKHOUSE_TMP}"/maps + ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE maps" + cat "${CLICKHOUSE_TMP}"/maps | ${CLICKHOUSE_CLIENT} -q "INSERT INTO maps FORMAT Parquet" + ${CLICKHOUSE_CLIENT} --query="SELECT * FROM maps" +done + +${CLICKHOUSE_CLIENT} --query="DROP TABLE maps" diff --git a/tests/queries/0_stateless/00900_orc_tuples_load.reference b/tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.reference similarity index 63% rename from tests/queries/0_stateless/00900_orc_tuples_load.reference rename to tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.reference index cbf8ee819f7..12556ca2c7b 100644 --- a/tests/queries/0_stateless/00900_orc_tuples_load.reference +++ b/tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.reference @@ -1,2 +1,6 @@ +Arrow (1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] +Parquet +(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] +ORC (1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh b/tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh new file mode 100755 index 00000000000..ff16d3205ad --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS tuples"; +${CLICKHOUSE_CLIENT} --query="CREATE TABLE tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" + +formats="Arrow Parquet ORC"; + +for format in ${formats}; do + echo $format + + ${CLICKHOUSE_CLIENT} --query="SELECT * FROM tuples FORMAT $format" > "${CLICKHOUSE_TMP}"/tuples + ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE tuples" + cat "${CLICKHOUSE_TMP}"/tuples | ${CLICKHOUSE_CLIENT} -q "INSERT INTO tuples FORMAT $format" + ${CLICKHOUSE_CLIENT} --query="SELECT * FROM tuples" +done + +${CLICKHOUSE_CLIENT} --query="DROP TABLE tuples" diff --git a/tests/queries/0_stateless/00900_orc_map_load.sh b/tests/queries/0_stateless/00900_orc_map_load.sh new file mode 100644 index 00000000000..3ddef17bd5c --- /dev/null +++ b/tests/queries/0_stateless/00900_orc_map_load.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_maps" +${CLICKHOUSE_CLIENT} --multiquery < "${CLICKHOUSE_TMP}"/maps.parquet + +cat "${CLICKHOUSE_TMP}"/maps.parquet | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_maps FORMAT Parquet" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_maps" +${CLICKHOUSE_CLIENT} --query="DROP TABLE parquet_maps" diff --git a/tests/queries/0_stateless/00900_orc_tuples_load.sh b/tests/queries/0_stateless/00900_orc_tuples_load.sh deleted file mode 100755 index f32be6af751..00000000000 --- a/tests/queries/0_stateless/00900_orc_tuples_load.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_tuples" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" - -${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_tuples FORMAT ORC" > "${CLICKHOUSE_TMP}"/tuples.orc - -cat "${CLICKHOUSE_TMP}"/tuples.orc | ${CLICKHOUSE_CLIENT} -q "INSERT INTO orc_tuples FORMAT ORC" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_tuples" -${CLICKHOUSE_CLIENT} --query="DROP TABLE orc_tuples" diff --git a/tests/queries/0_stateless/00900_parquet_tuples_load.reference b/tests/queries/0_stateless/00900_parquet_tuples_load.reference deleted file mode 100644 index cbf8ee819f7..00000000000 --- a/tests/queries/0_stateless/00900_parquet_tuples_load.reference +++ /dev/null @@ -1,2 +0,0 @@ -(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] -(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/00900_parquet_tuples_load.sh b/tests/queries/0_stateless/00900_parquet_tuples_load.sh deleted file mode 100755 index 031e8111019..00000000000 --- a/tests/queries/0_stateless/00900_parquet_tuples_load.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_tuples" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" - -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_tuples FORMAT Parquet" > "${CLICKHOUSE_TMP}"/tuples.parquet - -cat "${CLICKHOUSE_TMP}"/tuples.parquet | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_tuples FORMAT Parquet" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_tuples" -${CLICKHOUSE_CLIENT} --query="DROP TABLE parquet_tuples" diff --git a/tests/queries/0_stateless/01273_arrow_tuples_load.reference b/tests/queries/0_stateless/01273_arrow_tuples_load.reference deleted file mode 100644 index cbf8ee819f7..00000000000 --- a/tests/queries/0_stateless/01273_arrow_tuples_load.reference +++ /dev/null @@ -1,2 +0,0 @@ -(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] -(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))] diff --git a/tests/queries/0_stateless/01273_arrow_tuples_load.sh b/tests/queries/0_stateless/01273_arrow_tuples_load.sh deleted file mode 100755 index 311079afe50..00000000000 --- a/tests/queries/0_stateless/01273_arrow_tuples_load.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_tuples" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()" - -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_tuples FORMAT Arrow" > "${CLICKHOUSE_TMP}"/tuples.arrow - -cat "${CLICKHOUSE_TMP}"/tuples.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_tuples FORMAT Arrow" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_tuples" -${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_tuples" diff --git a/tests/queries/0_stateless/data_parquet/nested_maps.snappy.parquet.columns b/tests/queries/0_stateless/data_parquet/nested_maps.snappy.parquet.columns index d5e9599431b..eef66ae66c7 100644 --- a/tests/queries/0_stateless/data_parquet/nested_maps.snappy.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/nested_maps.snappy.parquet.columns @@ -1 +1 @@ -`a` Tuple(Nullable(String), Nullable(Int32), Nullable(UInt8)), `b` Nullable(Int32), `c` Nullable(Float64) +`a` Map(String, Map(Int32, Nullable(UInt8))), `b` Nullable(Int32), `c` Nullable(Float64) diff --git a/tests/queries/0_stateless/data_parquet/nonnullable.impala.parquet.columns b/tests/queries/0_stateless/data_parquet/nonnullable.impala.parquet.columns index 6d724200aec..299ec3b6af2 100644 --- a/tests/queries/0_stateless/data_parquet/nonnullable.impala.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/nonnullable.impala.parquet.columns @@ -1 +1 @@ -`ID` Nullable(Int64), `Int_Array` Nullable(Int32), `int_array_array` Nullable(Int32), `Int_Map` Tuple(Nullable(String), Nullable(Int32)), `int_map_array` Tuple(Nullable(String), Nullable(Int32)), `nested_Struct` Tuple(Nullable(Int32), Nullable(Int32), Nullable(Int32), Nullable(String), Nullable(String), Nullable(Float64)) +`ID` Nullable(Int64), `Int_Array` Array(Nullable(Int32)), `int_array_array` Array(Array(Nullable(Int32))), `Int_Map` Map(String, Nullable(Int32)), `int_map_array` Array(Map(String, Nullable(Int32))), `nested_Struct` Tuple(Nullable(Int32), Array(Nullable(Int32)), Tuple(Array(Array(Tuple(Nullable(Int32))))), Map(String, Tuple(Tuple(Array(Nullable(Float64)))))) diff --git a/tests/queries/0_stateless/data_parquet/nullable.impala.parquet.columns b/tests/queries/0_stateless/data_parquet/nullable.impala.parquet.columns index b5e122585d7..6fcbcdd4a0b 100644 --- a/tests/queries/0_stateless/data_parquet/nullable.impala.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/nullable.impala.parquet.columns @@ -1 +1 @@ -`id` Nullable(Int64), `int_array` Nullable(Int32), `int_array_Array` Nullable(Int32), `int_map` Tuple(Nullable(String), Nullable(Int32)), `int_Map_Array` Tuple(Nullable(String), Nullable(Int32)), `nested_struct` Tuple(Nullable(Int32), Nullable(Int32), Nullable(Int32), Nullable(String), Nullable(String), Nullable(Float64)) +`id` Nullable(Int64), `int_array` Array(Nullable(Int32)), `int_array_Array` Array(Array(Nullable(Int32))), `int_map` Map(String, Nullable(Int32)), `int_Map_Array` Array(Map(String, Nullable(Int32))), `nested_struct` Tuple(Nullable(Int32), Array(Nullable(Int32)), Tuple(Array(Array(Tuple(Nullable(Int32))))), Map(String, Tuple(Tuple(Array(Nullable(Float64)))))) diff --git a/tests/queries/0_stateless/data_parquet/nulls.snappy.parquet.columns b/tests/queries/0_stateless/data_parquet/nulls.snappy.parquet.columns index a99b8b80eac..6e723ef72c3 100644 --- a/tests/queries/0_stateless/data_parquet/nulls.snappy.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/nulls.snappy.parquet.columns @@ -1 +1 @@ -`b_struct` Nullable(Int32) +`b_struct` Tuple(Nullable(Int32)) From b120841b579662f97734667d2c59e98382b8c669 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 27 May 2021 22:10:45 +0300 Subject: [PATCH 201/491] Small changes --- src/DataTypes/DataTypeLowCardinality.h | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 2 +- src/Processors/Formats/Impl/ORCBlockInputFormat.cpp | 5 +---- .../Formats/Impl/ParquetBlockInputFormat.cpp | 13 ++++++------- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/DataTypeLowCardinality.h index 2ab62fabf41..1266174c6d6 100644 --- a/src/DataTypes/DataTypeLowCardinality.h +++ b/src/DataTypes/DataTypeLowCardinality.h @@ -13,7 +13,7 @@ private: DataTypePtr dictionary_type; public: - DataTypeLowCardinality(DataTypePtr dictionary_type_dict); + DataTypeLowCardinality(DataTypePtr dictionary_type_); const DataTypePtr & getDictionaryType() const { return dictionary_type; } diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 8f76f5dca8a..755cc03fa84 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -454,7 +454,7 @@ namespace DB const DataTypeArray * array_type = typeid_cast(column_type.get()); if (!array_type) - throw Exception{"Cannot convert arrow LIST type to a not Array/Map ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + throw Exception{"Cannot convert arrow LIST type to a not Array ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; return std::make_shared(getInternalType(list_nested_type, array_type->getNestedType(), column_name, format_name)); } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index cb503def801..0f8ca728c46 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -83,11 +83,8 @@ static size_t countIndicesForType(std::shared_ptr type) if (type->id() == arrow::Type::MAP) { - int indices = 0; auto * map_type = static_cast(type.get()); - indices += countIndicesForType(map_type->key_type()); - indices += countIndicesForType(map_type->item_type()); - return indices; + return countIndicesForType(map_type->key_type()) + countIndicesForType(map_type->item_type()); } return 1; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index bde39446dbb..ea7d35f5bbe 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -72,23 +72,22 @@ static size_t countIndicesForType(std::shared_ptr type) if (type->id() == arrow::Type::LIST) return countIndicesForType(static_cast(type.get())->value_type()); - int indices = 0; if (type->id() == arrow::Type::STRUCT) { + int indices = 0; auto * struct_type = static_cast(type.get()); for (int i = 0; i != struct_type->num_fields(); ++i) indices += countIndicesForType(struct_type->field(i)->type()); + return indices; } - else if (type->id() == arrow::Type::MAP) + + if (type->id() == arrow::Type::MAP) { auto * map_type = static_cast(type.get()); - indices += countIndicesForType(map_type->key_type()); - indices += countIndicesForType(map_type->item_type()); + return countIndicesForType(map_type->key_type()) + countIndicesForType(map_type->item_type()); } - else - indices = 1; - return indices; + return 1; } void ParquetBlockInputFormat::prepareReader() From 787c8000641a44e0bfe35de471ade5d7160c780f Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 1 Jun 2021 10:37:05 +0300 Subject: [PATCH 202/491] Fix build, tests, style --- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 20 ++++++++++--------- .../Formats/Impl/ArrowColumnToCHColumn.h | 4 ++-- .../Formats/Impl/CHColumnToArrowColumn.cpp | 7 ++++--- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 1 - 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 755cc03fa84..4fa5e190b6c 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -33,6 +33,7 @@ namespace DB extern const int CANNOT_CONVERT_TYPE; extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN; extern const int THERE_IS_NO_COLUMN; + extern const int BAD_ARGUMENTS; } static const std::initializer_list> arrow_type_to_internal_type = @@ -269,19 +270,21 @@ namespace DB offsets_data.emplace_back(start + arrow_offsets.Value(i)); } } - - static DataTypePtr getInternalIndexesType(std::shared_ptr arrow_type) + static ColumnPtr createAndFillColumnWithIndexesData(std::shared_ptr & arrow_column) { - switch (arrow_type->id()) + switch (arrow_column->type()->id()) { # define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \ case ARROW_NUMERIC_TYPE: \ - return std::make_shared>(); \ - - FOR_ARROW_INXEXES_TYPES(DISPATCH) + { \ + auto column = DataTypeNumber().createColumn(); \ + fillColumnWithNumericData(arrow_column, *column); \ + return column; \ + } + FOR_ARROW_INDEXES_TYPES(DISPATCH) # undef DISPATCH default: - throw Exception("Unsupported type for indexes in LowCardinality: " + arrow_type->name() + ".", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Unsupported type for indexes in LowCardinality: " + arrow_column->type()->name() + ".", ErrorCodes::BAD_ARGUMENTS); } } @@ -406,9 +409,8 @@ namespace DB } auto arrow_indexes_column = std::make_shared(indexes_array); - auto indexes_column = getInternalIndexesType(arrow_indexes_column->type())->createColumn(); + auto indexes_column = createAndFillColumnWithIndexesData(arrow_indexes_column); - readColumnFromArrowColumn(arrow_indexes_column, *indexes_column, column_name, format_name, is_nullable, dictionary_values); auto new_column_lc = ColumnLowCardinality::create(dict_values, std::move(indexes_column)); column_lc = std::move(*new_column_lc); break; diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index 60fbc30e9c9..0ba2d891a2c 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -36,7 +36,7 @@ namespace DB M(arrow::Type::FLOAT, DB::Float32) \ M(arrow::Type::DOUBLE, DB::Float64) -# define FOR_ARROW_INXEXES_TYPES(M) \ +# define FOR_ARROW_INDEXES_TYPES(M) \ M(arrow::Type::UINT8, DB::UInt8) \ M(arrow::Type::INT8, DB::UInt8) \ M(arrow::Type::UINT16, DB::UInt16) \ @@ -44,7 +44,7 @@ namespace DB M(arrow::Type::UINT32, DB::UInt32) \ M(arrow::Type::INT32, DB::UInt32) \ M(arrow::Type::UINT64, DB::UInt64) \ - M(arrow::Type::INT64, DB::UInt64) \ + M(arrow::Type::INT64, DB::UInt64) /// Map {column name : dictionary column}. /// To avoid converting dictionary from Arrow Dictionary diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 7041193e09a..c0909ac3d22 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -30,6 +30,7 @@ namespace DB { extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_TYPE; + extern const int LOGICAL_ERROR; } static const std::initializer_list>> internal_type_to_arrow_type = @@ -215,7 +216,7 @@ namespace DB /// Convert dictionary from LowCardinality to Arrow dictionary only once and then reuse it. if (!dict_values) { - const auto & value_type = assert_cast(builder->type().get())->value_type(); + auto value_type = assert_cast(builder->type().get())->value_type(); std::unique_ptr values_builder; arrow::MemoryPool* pool = arrow::default_memory_pool(); arrow::Status status = MakeBuilder(pool, value_type, &values_builder); @@ -550,8 +551,8 @@ namespace DB { auto nested_type = assert_cast(column_type.get())->getDictionaryType(); const auto * lc_column = assert_cast(column.get()); - ColumnPtr nested_column = lc_column->getDictionaryPtr(); - ColumnPtr indexes_column = lc_column->getIndexesPtr(); + const auto & nested_column = lc_column->getDictionaryPtr(); + const auto & indexes_column = lc_column->getIndexesPtr(); return arrow::dictionary( getArrowTypeForLowCardinalityIndexes(indexes_column), getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable)); diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 25eade4bafa..1eab69239ca 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include "ArrowBufferedStreams.h" #include "CHColumnToArrowColumn.h" From 02e68655b423488b3d02ea63b4ba37b92c4096ec Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 1 Jun 2021 11:32:09 +0300 Subject: [PATCH 203/491] Update orc and arrow --- .gitmodules | 2 +- contrib/arrow | 2 +- contrib/arrow-cmake/CMakeLists.txt | 37 ++++++++++++++++++++++-------- contrib/orc | 2 +- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.gitmodules b/.gitmodules index ab7c8a7c94d..2e9ea26dcac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -103,7 +103,7 @@ url = https://github.com/ClickHouse-Extras/fastops [submodule "contrib/orc"] path = contrib/orc - url = https://github.com/apache/orc + url = https://github.com/ClickHouse-Extras/orc [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11.git diff --git a/contrib/arrow b/contrib/arrow index 616b3dc76a0..debf751a129 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 616b3dc76a0c8450b4027ded8a78e9619d7c845f +Subproject commit debf751a129bdda9ff4d1e895e08957ff77000a1 diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index deefb244beb..069b96dd006 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -188,6 +188,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/array/util.cc" "${LIBRARY_DIR}/array/validate.cc" + "${LIBRARY_DIR}/compute/api_aggregate.cc" "${LIBRARY_DIR}/compute/api_scalar.cc" "${LIBRARY_DIR}/compute/api_vector.cc" "${LIBRARY_DIR}/compute/cast.cc" @@ -198,8 +199,11 @@ set(ARROW_SRCS "${LIBRARY_DIR}/compute/kernels/aggregate_basic.cc" "${LIBRARY_DIR}/compute/kernels/aggregate_mode.cc" + "${LIBRARY_DIR}/compute/kernels/aggregate_quantile.cc" + "${LIBRARY_DIR}/compute/kernels/aggregate_tdigest.cc" "${LIBRARY_DIR}/compute/kernels/aggregate_var_std.cc" "${LIBRARY_DIR}/compute/kernels/codegen_internal.cc" + "${LIBRARY_DIR}/compute/kernels/hash_aggregate.cc" "${LIBRARY_DIR}/compute/kernels/scalar_arithmetic.cc" "${LIBRARY_DIR}/compute/kernels/scalar_boolean.cc" "${LIBRARY_DIR}/compute/kernels/scalar_cast_boolean.cc" @@ -243,6 +247,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/io/interfaces.cc" "${LIBRARY_DIR}/io/memory.cc" "${LIBRARY_DIR}/io/slow.cc" + "${LIBRARY_DIR}/io/transform.cc" "${LIBRARY_DIR}/tensor/coo_converter.cc" "${LIBRARY_DIR}/tensor/csf_converter.cc" @@ -256,11 +261,8 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/bitmap_builders.cc" "${LIBRARY_DIR}/util/bitmap_ops.cc" "${LIBRARY_DIR}/util/bpacking.cc" + "${LIBRARY_DIR}/util/cancel.cc" "${LIBRARY_DIR}/util/compression.cc" - "${LIBRARY_DIR}/util/compression_lz4.cc" - "${LIBRARY_DIR}/util/compression_snappy.cc" - "${LIBRARY_DIR}/util/compression_zlib.cc" - "${LIBRARY_DIR}/util/compression_zstd.cc" "${LIBRARY_DIR}/util/cpu_info.cc" "${LIBRARY_DIR}/util/decimal.cc" "${LIBRARY_DIR}/util/delimiting.cc" @@ -268,20 +270,33 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/future.cc" "${LIBRARY_DIR}/util/int_util.cc" "${LIBRARY_DIR}/util/io_util.cc" - "${LIBRARY_DIR}/util/iterator.cc" "${LIBRARY_DIR}/util/key_value_metadata.cc" "${LIBRARY_DIR}/util/logging.cc" "${LIBRARY_DIR}/util/memory.cc" + "${LIBRARY_DIR}/util/mutex.cc" "${LIBRARY_DIR}/util/string_builder.cc" "${LIBRARY_DIR}/util/string.cc" "${LIBRARY_DIR}/util/task_group.cc" + "${LIBRARY_DIR}/util/tdigest.cc" "${LIBRARY_DIR}/util/thread_pool.cc" "${LIBRARY_DIR}/util/time.cc" "${LIBRARY_DIR}/util/trie.cc" + "${LIBRARY_DIR}/util/uri.cc" "${LIBRARY_DIR}/util/utf8.cc" "${LIBRARY_DIR}/util/value_parsing.cc" "${LIBRARY_DIR}/vendored/base64.cpp" + "${LIBRARY_DIR}/vendored/datetime/tz.cpp" + "${LIBRARY_DIR}/vendored/double-conversion/bignum.cc" + "${LIBRARY_DIR}/vendored/double-conversion/double-conversion.cc" + "${LIBRARY_DIR}/vendored/double-conversion/bignum-dtoa.cc" + "${LIBRARY_DIR}/vendored/double-conversion/fast-dtoa.cc" + "${LIBRARY_DIR}/vendored/double-conversion/cached-powers.cc" + "${LIBRARY_DIR}/vendored/double-conversion/fixed-dtoa.cc" + "${LIBRARY_DIR}/vendored/double-conversion/diy-fp.cc" + "${LIBRARY_DIR}/vendored/double-conversion/strtod.cc" + + "${LIBRARY_DIR}/c/bridge.cc" ${ORC_SRCS} ) @@ -368,14 +383,14 @@ set(PARQUET_SRCS "${LIBRARY_DIR}/column_reader.cc" "${LIBRARY_DIR}/column_scanner.cc" "${LIBRARY_DIR}/column_writer.cc" - "${LIBRARY_DIR}/deprecated_io.cc" "${LIBRARY_DIR}/encoding.cc" - "${LIBRARY_DIR}/encryption.cc" - "${LIBRARY_DIR}/encryption_internal.cc" + "${LIBRARY_DIR}/encryption/encryption.cc" + "${LIBRARY_DIR}/encryption/encryption_internal.cc" + "${LIBRARY_DIR}/encryption/internal_file_decryptor.cc" + "${LIBRARY_DIR}/encryption/internal_file_encryptor.cc" + "${LIBRARY_DIR}/exception.cc" "${LIBRARY_DIR}/file_reader.cc" "${LIBRARY_DIR}/file_writer.cc" - "${LIBRARY_DIR}/internal_file_decryptor.cc" - "${LIBRARY_DIR}/internal_file_encryptor.cc" "${LIBRARY_DIR}/level_conversion.cc" "${LIBRARY_DIR}/level_comparison.cc" "${LIBRARY_DIR}/metadata.cc" @@ -385,6 +400,8 @@ set(PARQUET_SRCS "${LIBRARY_DIR}/properties.cc" "${LIBRARY_DIR}/schema.cc" "${LIBRARY_DIR}/statistics.cc" + "${LIBRARY_DIR}/stream_reader.cc" + "${LIBRARY_DIR}/stream_writer.cc" "${LIBRARY_DIR}/types.cc" "${GEN_LIBRARY_DIR}/parquet_constants.cpp" diff --git a/contrib/orc b/contrib/orc index 5981208e394..0a936f6bbdb 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit 5981208e39447df84827f6a961d1da76bacb6078 +Subproject commit 0a936f6bbdb9303308973073f8623b5a8d82eae1 From eb86721d14fae27786058740dd55282440839d3c Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 1 Jun 2021 19:42:05 +0300 Subject: [PATCH 204/491] Fix style, output strings as binary instead of utf8 --- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 8 +++++--- src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 4fa5e190b6c..2dad3a15026 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -472,10 +472,12 @@ namespace DB int internal_fields_num = tuple_nested_types.size(); /// If internal column has less elements then arrow struct, we will select only first internal_fields_num columns. if (internal_fields_num > struct_type->num_fields()) - throw Exception{ - "Cannot convert arrow STRUCT with " + std::to_string(struct_type->num_fields()) + " fields to a ClickHouse Tuple with " + throw Exception + { + "Cannot convert arrow STRUCT with " + std::to_string(struct_type->num_fields()) + " fields to a ClickHouse Tuple with " + std::to_string(internal_fields_num) + " elements " + column_type->getName(), - ErrorCodes::CANNOT_CONVERT_TYPE}; + ErrorCodes::CANNOT_CONVERT_TYPE + }; DataTypes nested_types; for (int i = 0; i < internal_fields_num; ++i) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index c0909ac3d22..c99c80c8d84 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -52,9 +52,8 @@ namespace DB //{"DateTime", arrow::date64()}, // BUG! saves as date32 {"DateTime", arrow::uint32()}, - // TODO: ClickHouse can actually store non-utf8 strings! - {"String", arrow::utf8()}, - {"FixedString", arrow::utf8()}, + {"String", arrow::binary()}, + {"FixedString", arrow::binary()}, }; @@ -287,7 +286,7 @@ namespace DB size_t end) { const auto & internal_column = assert_cast(*write_column); - arrow::StringBuilder & builder = assert_cast(*array_builder); + arrow::BinaryBuilder & builder = assert_cast(*array_builder); arrow::Status status; for (size_t string_i = start; string_i < end; ++string_i) From 83c843a20bb667bd32030cedf8e23b11cdcf3939 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 2 Jun 2021 00:47:18 +0300 Subject: [PATCH 205/491] Fix tests and build --- contrib/arrow-cmake/CMakeLists.txt | 1 - .../queries/0_stateless/00900_orc_map_load.sh | 20 ------------------- .../01273_arrow_dictionaries_load.reference | 2 -- 3 files changed, 23 deletions(-) delete mode 100644 tests/queries/0_stateless/00900_orc_map_load.sh diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 069b96dd006..878b46b39a6 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -281,7 +281,6 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/thread_pool.cc" "${LIBRARY_DIR}/util/time.cc" "${LIBRARY_DIR}/util/trie.cc" - "${LIBRARY_DIR}/util/uri.cc" "${LIBRARY_DIR}/util/utf8.cc" "${LIBRARY_DIR}/util/value_parsing.cc" diff --git a/tests/queries/0_stateless/00900_orc_map_load.sh b/tests/queries/0_stateless/00900_orc_map_load.sh deleted file mode 100644 index 3ddef17bd5c..00000000000 --- a/tests/queries/0_stateless/00900_orc_map_load.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_maps" -${CLICKHOUSE_CLIENT} --multiquery < "${CLICKHOUSE_TMP}"/maps.parquet - -cat "${CLICKHOUSE_TMP}"/maps.parquet | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_maps FORMAT Parquet" - -${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_maps" -${CLICKHOUSE_CLIENT} --query="DROP TABLE parquet_maps" diff --git a/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference b/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference index b144130918f..7321c396a59 100644 --- a/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference +++ b/tests/queries/0_stateless/01273_arrow_dictionaries_load.reference @@ -1,4 +1,2 @@ 1 ['a','b','c'] ('z','6') 2 ['d','e'] ('x','9') -1 ['a','b','c'] ('z','6') -2 ['d','e'] ('x','9') From a4ef60e230d9244a49831486f9bdcf24281d02da Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 2 Jun 2021 11:51:07 +0300 Subject: [PATCH 206/491] Remove Impl including from .h file --- src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp | 5 +++-- src/Processors/Formats/Impl/ArrowBlockInputFormat.h | 4 ++-- src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp | 8 ++++++-- src/Processors/Formats/Impl/ArrowBlockOutputFormat.h | 5 +++-- src/Processors/Formats/Impl/ORCBlockInputFormat.cpp | 4 ++-- src/Processors/Formats/Impl/ORCBlockInputFormat.h | 6 ++++-- src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp | 4 ++-- src/Processors/Formats/Impl/ParquetBlockInputFormat.h | 5 +++-- src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp | 4 ++-- src/Processors/Formats/Impl/ParquetBlockOutputFormat.h | 6 ++++-- 10 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 0ac8251b8bb..ce34fdfdc58 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -1,4 +1,5 @@ #include "ArrowBlockInputFormat.h" + #if USE_ARROW #include @@ -22,7 +23,7 @@ namespace ErrorCodes } ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_) - : IInputFormat(header_, in_), stream{stream_} + : IInputFormat(header_, in_), stream{stream_}, arrow_column_to_ch_column(std::make_unique()) { } @@ -63,7 +64,7 @@ Chunk ArrowBlockInputFormat::generate() ++record_batch_current; - arrow_column_to_ch_column.arrowTableToCHChunk(res, *table_result, header, "Arrow"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, header, "Arrow"); return res; } diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 6a8acc4a118..3bfead93bf1 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -3,7 +3,6 @@ #if USE_ARROW #include -#include namespace arrow { class RecordBatchReader; } namespace arrow::ipc { class RecordBatchFileReader; } @@ -12,6 +11,7 @@ namespace DB { class ReadBuffer; +class ArrowColumnToCHColumn; class ArrowBlockInputFormat : public IInputFormat { @@ -33,7 +33,7 @@ private: // The following fields are used only for Arrow format std::shared_ptr file_reader; - ArrowColumnToCHColumn arrow_column_to_ch_column; + std::unique_ptr arrow_column_to_ch_column; int record_batch_total = 0; int record_batch_current = 0; diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index d1fdffb700c..9f619320b73 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -18,7 +18,11 @@ namespace ErrorCodes } ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_) - : IOutputFormat(header_, out_), stream{stream_}, format_settings{format_settings_}, arrow_ostream{std::make_shared(out_)} + : IOutputFormat(header_, out_) + , stream{stream_} + , format_settings{format_settings_} + , arrow_ostream{std::make_shared(out_)} + , ch_column_to_arrow_column(std::make_unique()) { } @@ -28,7 +32,7 @@ void ArrowBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - ch_column_to_arrow_column.chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); if (!writer) prepareWriter(arrow_table->schema()); diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h index fc8efe62435..40d81f8b919 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h @@ -4,7 +4,6 @@ #include #include -#include #include "ArrowBufferedStreams.h" namespace arrow { class Schema; } @@ -13,6 +12,8 @@ namespace arrow::ipc { class RecordBatchWriter; } namespace DB { +class CHColumnToArrowColumn; + class ArrowBlockOutputFormat : public IOutputFormat { public: @@ -29,7 +30,7 @@ private: const FormatSettings format_settings; std::shared_ptr arrow_ostream; std::shared_ptr writer; - CHColumnToArrowColumn ch_column_to_arrow_column; + std::unique_ptr ch_column_to_arrow_column; void prepareWriter(const std::shared_ptr & schema); }; diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 0f8ca728c46..bd427bd62e1 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -26,7 +26,7 @@ namespace ErrorCodes throw Exception(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ } while (false) -ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer & in_, Block header_) : IInputFormat(std::move(header_), in_) +ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer & in_, Block header_) : IInputFormat(std::move(header_), in_), arrow_column_to_ch_column(std::make_unique()) { } @@ -54,7 +54,7 @@ Chunk ORCBlockInputFormat::generate() ++stripe_current; - arrow_column_to_ch_column.arrowTableToCHChunk(res, *table_result, header, "ORC"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, header, "ORC"); return res; } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index 5a6cfd1364a..f27685a9884 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -3,12 +3,14 @@ #if USE_ORC #include -#include namespace arrow::adapters::orc { class ORCFileReader; } namespace DB { + +class ArrowColumnToCHColumn; + class ORCBlockInputFormat : public IInputFormat { public: @@ -27,7 +29,7 @@ private: std::unique_ptr file_reader; - ArrowColumnToCHColumn arrow_column_to_ch_column; + std::unique_ptr arrow_column_to_ch_column; int stripe_total = 0; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index ea7d35f5bbe..c0d9e330df2 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -31,7 +31,7 @@ namespace ErrorCodes } while (false) ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_) - : IInputFormat(std::move(header_), in_) + : IInputFormat(std::move(header_), in_), arrow_column_to_ch_column(std::make_unique()) { } @@ -54,7 +54,7 @@ Chunk ParquetBlockInputFormat::generate() ++row_group_current; - arrow_column_to_ch_column.arrowTableToCHChunk(res, table, header, "Parquet"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, table, header, "Parquet"); return res; } diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index b27bafe04bf..b68f97c005a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -4,7 +4,6 @@ #if USE_PARQUET #include -#include namespace parquet::arrow { class FileReader; } @@ -13,6 +12,8 @@ namespace arrow { class Buffer; } namespace DB { +class ArrowColumnToCHColumn; + class ParquetBlockInputFormat : public IInputFormat { public: @@ -33,7 +34,7 @@ private: int row_group_total = 0; // indices of columns to read from Parquet file std::vector column_indices; - ArrowColumnToCHColumn arrow_column_to_ch_column; + std::unique_ptr arrow_column_to_ch_column; int row_group_current = 0; }; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 1eab69239ca..96ef6702cc4 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -25,7 +25,7 @@ namespace ErrorCodes } ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) - : IOutputFormat(header_, out_), format_settings{format_settings_} + : IOutputFormat(header_, out_), format_settings{format_settings_}, ch_column_to_arrow_column(std::make_unique()) { } @@ -35,7 +35,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - ch_column_to_arrow_column.chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet"); + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet"); if (!file_writer) { diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index deb011e0274..8114d1ab494 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -4,7 +4,6 @@ #if USE_PARQUET # include # include -# include namespace arrow { @@ -22,6 +21,9 @@ namespace arrow namespace DB { + +class CHColumnToArrowColumn; + class ParquetBlockOutputFormat : public IOutputFormat { public: @@ -37,7 +39,7 @@ private: const FormatSettings format_settings; std::unique_ptr file_writer; - CHColumnToArrowColumn ch_column_to_arrow_column; + std::unique_ptr ch_column_to_arrow_column; }; } From 57a9659c681f921cbbb7409f6a08bacea453bc18 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 3 Jun 2021 16:03:22 +0300 Subject: [PATCH 207/491] Add Arrow/Parquet/ORC in perf tests --- tests/performance/parse_engine_file.xml | 3 +++ tests/performance/select_format.xml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/performance/parse_engine_file.xml b/tests/performance/parse_engine_file.xml index 2740b680b67..2b67c19a4f6 100644 --- a/tests/performance/parse_engine_file.xml +++ b/tests/performance/parse_engine_file.xml @@ -22,6 +22,9 @@ Native Avro MsgPack + ORC + Parquet + Arrow diff --git a/tests/performance/select_format.xml b/tests/performance/select_format.xml index 985ec0f2b52..982039102d0 100644 --- a/tests/performance/select_format.xml +++ b/tests/performance/select_format.xml @@ -36,6 +36,8 @@ Avro MsgPack ORC + Parquet + Arrow From 931e05ab047231fa0af681f941a278b04790cf40 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Mon, 7 Jun 2021 18:15:58 +0300 Subject: [PATCH 208/491] Minor refactoring --- .../Formats/Impl/ArrowBlockInputFormat.cpp | 11 +++- .../Formats/Impl/ArrowBlockOutputFormat.cpp | 6 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 59 +++++++++++-------- .../Formats/Impl/ArrowColumnToCHColumn.h | 31 +++++----- .../Formats/Impl/CHColumnToArrowColumn.cpp | 56 ++++++++++-------- .../Formats/Impl/CHColumnToArrowColumn.h | 12 ++-- .../Formats/Impl/ORCBlockInputFormat.cpp | 7 ++- .../Formats/Impl/ParquetBlockInputFormat.cpp | 7 ++- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 6 +- 9 files changed, 117 insertions(+), 78 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index ce34fdfdc58..269faac5258 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -23,14 +23,13 @@ namespace ErrorCodes } ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_) - : IInputFormat(header_, in_), stream{stream_}, arrow_column_to_ch_column(std::make_unique()) + : IInputFormat(header_, in_), stream{stream_} { } Chunk ArrowBlockInputFormat::generate() { Chunk res; - const Block & header = getPort().getHeader(); arrow::Result> batch_result; if (stream) @@ -64,7 +63,7 @@ Chunk ArrowBlockInputFormat::generate() ++record_batch_current; - arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, header, "Arrow"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result); return res; } @@ -82,6 +81,8 @@ void ArrowBlockInputFormat::resetParser() void ArrowBlockInputFormat::prepareReader() { + std::shared_ptr schema; + if (stream) { auto stream_reader_status = arrow::ipc::RecordBatchStreamReader::Open(std::make_unique(in)); @@ -89,6 +90,7 @@ void ArrowBlockInputFormat::prepareReader() throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, "Error while opening a table: {}", stream_reader_status.status().ToString()); stream_reader = *stream_reader_status; + schema = stream_reader->schema(); } else { @@ -97,8 +99,11 @@ void ArrowBlockInputFormat::prepareReader() throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, "Error while opening a table: {}", file_reader_status.status().ToString()); file_reader = *file_reader_status; + schema = file_reader->schema(); } + arrow_column_to_ch_column = std::make_unique(getPort().getHeader(), std::move(schema), "Arrow"); + if (stream) record_batch_total = -1; else diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index 9f619320b73..42679298e07 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -22,7 +22,6 @@ ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & , stream{stream_} , format_settings{format_settings_} , arrow_ostream{std::make_shared(out_)} - , ch_column_to_arrow_column(std::make_unique()) { } @@ -32,10 +31,13 @@ void ArrowBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num); if (!writer) + { prepareWriter(arrow_table->schema()); + ch_column_to_arrow_column = std::make_unique(header, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); + } // TODO: calculate row_group_size depending on a number of rows and table size auto status = writer->WriteTable(*arrow_table, format_settings.arrow.row_group_size); diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 2dad3a15026..856915acf3c 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace DB @@ -159,11 +160,11 @@ namespace DB if (days_num > DATE_LUT_MAX_DAY_NUM) { // TODO: will it rollback correctly? - throw Exception{"Input value " + std::to_string(days_num) + " of a column \"" + internal_column.getName() - + "\" is greater than " - "max allowed Date value, which is " - + std::to_string(DATE_LUT_MAX_DAY_NUM), - ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE}; + throw Exception + { + fmt::format("Input value {} of a column \"{}\" is greater than max allowed Date value, which is {}", days_num, internal_column.getName(), DATE_LUT_MAX_DAY_NUM), + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE + }; } column_data.emplace_back(days_num); @@ -284,7 +285,7 @@ namespace DB FOR_ARROW_INDEXES_TYPES(DISPATCH) # undef DISPATCH default: - throw Exception("Unsupported type for indexes in LowCardinality: " + arrow_column->type()->name() + ".", ErrorCodes::BAD_ARGUMENTS); + throw Exception(fmt::format("Unsupported type for indexes in LowCardinality: {}.", arrow_column->type()->name()), ErrorCodes::BAD_ARGUMENTS); } } @@ -292,7 +293,7 @@ namespace DB std::shared_ptr & arrow_column, IColumn & internal_column, const std::string & column_name, - const std::string format_name, + const std::string & format_name, bool is_nullable, std::unordered_map dictionary_values) { @@ -310,7 +311,7 @@ namespace DB { throw Exception { - "Can not insert NULL data into non-nullable column \"" + column_name + "\"", + fmt::format("Can not insert NULL data into non-nullable column \"{}\".", column_name), ErrorCodes::CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN }; } @@ -335,7 +336,6 @@ namespace DB fillColumnWithTimestampData(arrow_column, internal_column); break; case arrow::Type::DECIMAL: - //fillColumnWithNumericData>(arrow_column, read_column); // Have problems with trash values under NULL, but faster fillColumnWithDecimalData(arrow_column, internal_column /*, internal_nested_type*/); break; case arrow::Type::MAP: [[fallthrough]]; @@ -428,8 +428,7 @@ namespace DB default: throw Exception { - "Unsupported " + format_name + " type \"" + arrow_column->type()->name() + "\" of an input column \"" - + column_name + "\"", + fmt::format("Unsupported {} type \"{}\" of an input column \"{}\".", format_name, arrow_column->type()->name(), column_name), ErrorCodes::UNKNOWN_TYPE }; } @@ -456,7 +455,7 @@ namespace DB const DataTypeArray * array_type = typeid_cast(column_type.get()); if (!array_type) - throw Exception{"Cannot convert arrow LIST type to a not Array ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + throw Exception{fmt::format("Cannot convert arrow LIST type to a not Array ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE}; return std::make_shared(getInternalType(list_nested_type, array_type->getNestedType(), column_name, format_name)); } @@ -466,7 +465,7 @@ namespace DB const auto * struct_type = static_cast(arrow_type.get()); const DataTypeTuple * tuple_type = typeid_cast(column_type.get()); if (!tuple_type) - throw Exception{"Cannot convert arrow STRUCT type to a not Tuple ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + throw Exception{fmt::format("Cannot convert arrow STRUCT type to a not Tuple ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE}; const DataTypes & tuple_nested_types = tuple_type->getElements(); int internal_fields_num = tuple_nested_types.size(); @@ -474,8 +473,11 @@ namespace DB if (internal_fields_num > struct_type->num_fields()) throw Exception { - "Cannot convert arrow STRUCT with " + std::to_string(struct_type->num_fields()) + " fields to a ClickHouse Tuple with " - + std::to_string(internal_fields_num) + " elements " + column_type->getName(), + fmt::format( + "Cannot convert arrow STRUCT with {} fields to a ClickHouse Tuple with {} elements: {}.", + struct_type->num_fields(), + internal_fields_num, + column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE }; @@ -500,7 +502,7 @@ namespace DB const auto * arrow_map_type = typeid_cast(arrow_type.get()); const auto * map_type = typeid_cast(column_type.get()); if (!map_type) - throw Exception{"Cannot convert arrow MAP type to a not Map ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE}; + throw Exception{fmt::format("Cannot convert arrow MAP type to a not Map ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE}; return std::make_shared( getInternalType(arrow_map_type->key_type(), map_type->getKeyType(), column_name, format_name), @@ -516,13 +518,24 @@ namespace DB } throw Exception { - "The type \"" + arrow_type->name() + "\" of an input column \"" + column_name + "\" is not supported for conversion from a " + format_name + " data format", + fmt::format("The type \"{}\" of an input column \"{}\" is not supported for conversion from a {} data format.", arrow_type->name(), column_name, format_name), ErrorCodes::CANNOT_CONVERT_TYPE }; } - void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, - const Block & header, std::string format_name) + ArrowColumnToCHColumn::ArrowColumnToCHColumn(const Block & header_, std::shared_ptr schema_, const std::string & format_name_) : header(header_), format_name(format_name_) + { + for (const auto & field : schema_->fields()) + { + if (header.has(field->name())) + { + const auto column_type = recursiveRemoveLowCardinality(header.getByName(field->name()).type); + name_to_internal_type[field->name()] = getInternalType(field->type(), column_type, field->name(), format_name); + } + } + } + + void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr & table) { Columns columns_list; UInt64 num_rows = 0; @@ -540,18 +553,16 @@ namespace DB for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i) { - ColumnWithTypeAndName header_column = header.getByPosition(column_i); - const auto column_type = recursiveRemoveLowCardinality(header_column.type); + const ColumnWithTypeAndName & header_column = header.getByPosition(column_i); if (name_to_column_ptr.find(header_column.name) == name_to_column_ptr.end()) // TODO: What if some columns were not presented? Insert NULLs? What if a column is not nullable? - throw Exception{"Column \"" + header_column.name + "\" is not presented in input data", + throw Exception{fmt::format("Column \"{}\" is not presented in input data.", header_column.name), ErrorCodes::THERE_IS_NO_COLUMN}; std::shared_ptr arrow_column = name_to_column_ptr[header_column.name]; - DataTypePtr internal_type = getInternalType(arrow_column->type(), column_type, header_column.name, format_name); - + DataTypePtr & internal_type = name_to_internal_type[header_column.name]; MutableColumnPtr read_column = internal_type->createColumn(); readColumnFromArrowColumn(arrow_column, *read_column, header_column.name, format_name, false, dictionary_values); diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index 0ba2d891a2c..7da54a8a02d 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -19,11 +19,15 @@ namespace DB { - class ArrowColumnToCHColumn - { - private: +class ArrowColumnToCHColumn +{ +public: + ArrowColumnToCHColumn(const Block & header_, std::shared_ptr schema_, const std::string & format_name_); -# define FOR_ARROW_NUMERIC_TYPES(M) \ + void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table); + +private: +#define FOR_ARROW_NUMERIC_TYPES(M) \ M(arrow::Type::UINT8, DB::UInt8) \ M(arrow::Type::INT8, DB::Int8) \ M(arrow::Type::UINT16, DB::UInt16) \ @@ -36,7 +40,7 @@ namespace DB M(arrow::Type::FLOAT, DB::Float32) \ M(arrow::Type::DOUBLE, DB::Float64) -# define FOR_ARROW_INDEXES_TYPES(M) \ +#define FOR_ARROW_INDEXES_TYPES(M) \ M(arrow::Type::UINT8, DB::UInt8) \ M(arrow::Type::INT8, DB::UInt8) \ M(arrow::Type::UINT16, DB::UInt16) \ @@ -46,15 +50,14 @@ namespace DB M(arrow::Type::UINT64, DB::UInt64) \ M(arrow::Type::INT64, DB::UInt64) - /// Map {column name : dictionary column}. - /// To avoid converting dictionary from Arrow Dictionary - /// to LowCardinality every chunk we save it and reuse. - std::unordered_map dictionary_values; - public: - - void arrowTableToCHChunk(Chunk & res, std::shared_ptr & table, - const Block & header, std::string format_name); - }; + const Block & header; + std::unordered_map name_to_internal_type; + const std::string format_name; + /// Map {column name : dictionary column}. + /// To avoid converting dictionary from Arrow Dictionary + /// to LowCardinality every chunk we save it and reuse. + std::unordered_map dictionary_values; +}; } #endif diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index c99c80c8d84..01fc80ad950 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -60,7 +60,7 @@ namespace DB static void checkStatus(const arrow::Status & status, const String & column_name, const String & format_name) { if (!status.ok()) - throw Exception{"Error with a " + format_name + " column \"" + column_name + "\": " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + throw Exception{fmt::format("Error with a {} column \"{}\": {}.", format_name, column_name, status.ToString()), ErrorCodes::UNKNOWN_EXCEPTION}; } template @@ -191,7 +191,7 @@ namespace DB case TypeIndex::UInt64: return extractIndexesImpl(column, start, end); default: - throw Exception("Indexes column must be ColumnUInt, got " + column->getName(), + throw Exception(fmt::format("Indexes column must be ColumnUInt, got {}.", column->getName()), ErrorCodes::LOGICAL_ERROR); } } @@ -439,7 +439,7 @@ namespace DB { throw Exception { - "Internal type \"" + column_type_name + "\" of a column \"" + column_name + "\" is not supported for conversion into a " + format_name + " data format", + fmt::format("Internal type \"{}\" of a column \"{}\" is not supported for conversion into a {} data format.", column_type_name, column_name, format_name), ErrorCodes::UNKNOWN_TYPE }; } @@ -486,7 +486,7 @@ namespace DB case TypeIndex::UInt64: return arrow::int64(); default: - throw Exception("Indexes column for getUniqueIndex must be ColumnUInt, got " + indexes_column->getName(), + throw Exception(fmt::format("Indexes column for getUniqueIndex must be ColumnUInt, got {}.", indexes_column->getName()), ErrorCodes::LOGICAL_ERROR); } } @@ -580,51 +580,61 @@ namespace DB return arrow_type_it->second; } - throw Exception{"The type \"" + column_type->getName() + "\" of a column \"" + column_name + "\"" - " is not supported for conversion into a " + format_name + " data format", + throw Exception{fmt::format("The type \"{}\" of a column \"{}\" is not supported for conversion into a {} data format.", column_type->getName(), column_name, format_name), ErrorCodes::UNKNOWN_TYPE}; } + CHColumnToArrowColumn::CHColumnToArrowColumn(const Block & header, const std::string & format_name_, bool low_cardinality_as_dictionary_) + : format_name(format_name_), low_cardinality_as_dictionary(low_cardinality_as_dictionary_) + { + arrow_fields.reserve(header.columns()); + header_columns.reserve(header.columns()); + for (auto column : header.getColumnsWithTypeAndName()) + { + if (!low_cardinality_as_dictionary) + { + column.type = recursiveRemoveLowCardinality(column.type); + column.column = recursiveRemoveLowCardinality(column.column); + } + bool is_column_nullable = false; + auto arrow_type = getArrowType(column.type, column.column, column.name, format_name, &is_column_nullable); + arrow_fields.emplace_back(std::make_shared(column.name, arrow_type, is_column_nullable)); + header_columns.emplace_back(std::move(column)); + } + } + void CHColumnToArrowColumn::chChunkToArrowTable( std::shared_ptr & res, - const Block & header, const Chunk & chunk, - size_t columns_num, - String format_name, - bool low_cardinality_as_dictionary) + size_t columns_num) { /// For arrow::Schema and arrow::Table creation - std::vector> arrow_fields; std::vector> arrow_arrays; arrow_fields.reserve(columns_num); arrow_arrays.reserve(columns_num); for (size_t column_i = 0; column_i < columns_num; ++column_i) { - // TODO: constructed every iteration - ColumnWithTypeAndName column = header.safeGetByPosition(column_i); - column.column = chunk.getColumns()[column_i]; + const ColumnWithTypeAndName & header_column = header_columns[column_i]; + auto column = chunk.getColumns()[column_i]; if (!low_cardinality_as_dictionary) - { - column.column = recursiveRemoveLowCardinality(column.column); - column.type = recursiveRemoveLowCardinality(column.type); - } + column = recursiveRemoveLowCardinality(column); bool is_column_nullable = false; - auto arrow_type = getArrowType(column.type, column.column, column.name, format_name, &is_column_nullable); - arrow_fields.emplace_back(std::make_shared(column.name, arrow_type, is_column_nullable)); + auto arrow_type = getArrowType(header_column.type, column, header_column.name, format_name, &is_column_nullable); + arrow_fields.emplace_back(std::make_shared(header_column.name, arrow_type, is_column_nullable)); arrow::MemoryPool* pool = arrow::default_memory_pool(); std::unique_ptr array_builder; arrow::Status status = MakeBuilder(pool, arrow_fields[column_i]->type(), &array_builder); - checkStatus(status, column.column->getName(), format_name); + checkStatus(status, column->getName(), format_name); - fillArrowArray(column.name, column.column, column.type, nullptr, array_builder.get(), format_name, 0, column.column->size(), dictionary_values); + fillArrowArray(header_column.name, column, header_column.type, nullptr, array_builder.get(), format_name, 0, column->size(), dictionary_values); std::shared_ptr arrow_array; status = array_builder->Finish(&arrow_array); - checkStatus(status, column.column->getName(), format_name); + checkStatus(status, column->getName(), format_name); arrow_arrays.emplace_back(std::move(arrow_array)); } diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h index 9df70b5bceb..efe02a0d7d9 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h @@ -12,6 +12,10 @@ namespace DB class CHColumnToArrowColumn { +public: + CHColumnToArrowColumn(const Block & header, const std::string & format_name_, bool low_cardinality_as_dictionary_ = false); + + void chChunkToArrowTable(std::shared_ptr & res, const Chunk & chunk, size_t columns_num); private: #define FOR_INTERNAL_NUMERIC_TYPES(M) \ @@ -39,14 +43,14 @@ private: M(DOUBLE, arrow::DoubleType) \ M(STRING, arrow::StringType) + ColumnsWithTypeAndName header_columns; + std::vector> arrow_fields; + const std::string format_name; + bool low_cardinality_as_dictionary; /// Map {column name : arrow dictionary}. /// To avoid converting dictionary from LowCardinality to Arrow /// Dictionary every chunk we save it and reuse. std::unordered_map> dictionary_values; - -public: - void chChunkToArrowTable(std::shared_ptr & res, const Block & header, const Chunk & chunk, - size_t columns_num, String format_name, bool low_cardinality_as_dictionary = false); }; } #endif diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index bd427bd62e1..6ee247413e9 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -26,14 +26,13 @@ namespace ErrorCodes throw Exception(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ } while (false) -ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer & in_, Block header_) : IInputFormat(std::move(header_), in_), arrow_column_to_ch_column(std::make_unique()) +ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer & in_, Block header_) : IInputFormat(std::move(header_), in_) { } Chunk ORCBlockInputFormat::generate() { Chunk res; - const Block & header = getPort().getHeader(); if (!file_reader) prepareReader(); @@ -54,7 +53,7 @@ Chunk ORCBlockInputFormat::generate() ++stripe_current; - arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result, header, "ORC"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result); return res; } @@ -99,6 +98,8 @@ void ORCBlockInputFormat::prepareReader() std::shared_ptr schema; THROW_ARROW_NOT_OK(file_reader->ReadSchema(&schema)); + arrow_column_to_ch_column = std::make_unique(getPort().getHeader(), schema, "ORC"); + /// In ReadStripe column indices should be started from 1, /// because 0 indicates to select all columns. int index = 1; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index c0d9e330df2..07a0e15cb6b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -31,14 +31,13 @@ namespace ErrorCodes } while (false) ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_) - : IInputFormat(std::move(header_), in_), arrow_column_to_ch_column(std::make_unique()) + : IInputFormat(std::move(header_), in_) { } Chunk ParquetBlockInputFormat::generate() { Chunk res; - const Block & header = getPort().getHeader(); if (!file_reader) prepareReader(); @@ -54,7 +53,7 @@ Chunk ParquetBlockInputFormat::generate() ++row_group_current; - arrow_column_to_ch_column->arrowTableToCHChunk(res, table, header, "Parquet"); + arrow_column_to_ch_column->arrowTableToCHChunk(res, table); return res; } @@ -99,6 +98,8 @@ void ParquetBlockInputFormat::prepareReader() std::shared_ptr schema; THROW_ARROW_NOT_OK(file_reader->GetSchema(&schema)); + arrow_column_to_ch_column = std::make_unique(getPort().getHeader(), schema, "Parquet"); + int index = 0; for (int i = 0; i < schema->num_fields(); ++i) { diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 96ef6702cc4..c53ee44de36 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -25,7 +25,7 @@ namespace ErrorCodes } ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) - : IOutputFormat(header_, out_), format_settings{format_settings_}, ch_column_to_arrow_column(std::make_unique()) + : IOutputFormat(header_, out_), format_settings{format_settings_} { } @@ -35,7 +35,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; - ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet"); + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num); if (!file_writer) { @@ -54,6 +54,8 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) &file_writer); if (!status.ok()) throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + + ch_column_to_arrow_column = std::make_unique(header, "Parquet"); } // TODO: calculate row_group_size depending on a number of rows and table size From 692150b578737f1230d7fd3fdaf6b9a11639703e Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 8 Jun 2021 13:37:54 +0300 Subject: [PATCH 209/491] Fix tests --- .../Formats/Impl/ArrowBlockOutputFormat.cpp | 11 +++++++---- src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp | 8 +------- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 9 ++++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index 42679298e07..8f43d03de38 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -27,17 +27,20 @@ ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & void ArrowBlockOutputFormat::consume(Chunk chunk) { - const Block & header = getPort(PortKind::Main).getHeader(); const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; + if (!ch_column_to_arrow_column) + { + const Block & header = getPort(PortKind::Main).getHeader(); + ch_column_to_arrow_column + = std::make_unique(header, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); + } + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num); if (!writer) - { prepareWriter(arrow_table->schema()); - ch_column_to_arrow_column = std::make_unique(header, "Arrow", format_settings.arrow.low_cardinality_as_dictionary); - } // TODO: calculate row_group_size depending on a number of rows and table size auto status = writer->WriteTable(*arrow_table, format_settings.arrow.row_group_size); diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 01fc80ad950..4dbb95b327a 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -610,9 +610,7 @@ namespace DB { /// For arrow::Schema and arrow::Table creation std::vector> arrow_arrays; - arrow_fields.reserve(columns_num); arrow_arrays.reserve(columns_num); - for (size_t column_i = 0; column_i < columns_num; ++column_i) { const ColumnWithTypeAndName & header_column = header_columns[column_i]; @@ -621,10 +619,6 @@ namespace DB if (!low_cardinality_as_dictionary) column = recursiveRemoveLowCardinality(column); - bool is_column_nullable = false; - auto arrow_type = getArrowType(header_column.type, column, header_column.name, format_name, &is_column_nullable); - arrow_fields.emplace_back(std::make_shared(header_column.name, arrow_type, is_column_nullable)); - arrow::MemoryPool* pool = arrow::default_memory_pool(); std::unique_ptr array_builder; arrow::Status status = MakeBuilder(pool, arrow_fields[column_i]->type(), &array_builder); @@ -638,7 +632,7 @@ namespace DB arrow_arrays.emplace_back(std::move(arrow_array)); } - std::shared_ptr arrow_schema = std::make_shared(std::move(arrow_fields)); + std::shared_ptr arrow_schema = std::make_shared(arrow_fields); res = arrow::Table::Make(arrow_schema, arrow_arrays); } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index c53ee44de36..800fd0ff0e8 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -31,10 +31,15 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Blo void ParquetBlockOutputFormat::consume(Chunk chunk) { - const Block & header = getPort(PortKind::Main).getHeader(); const size_t columns_num = chunk.getNumColumns(); std::shared_ptr arrow_table; + if (!ch_column_to_arrow_column) + { + const Block & header = getPort(PortKind::Main).getHeader(); + ch_column_to_arrow_column = std::make_unique(header, "Parquet"); + } + ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num); if (!file_writer) @@ -54,8 +59,6 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) &file_writer); if (!status.ok()) throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; - - ch_column_to_arrow_column = std::make_unique(header, "Parquet"); } // TODO: calculate row_group_size depending on a number of rows and table size From 313b0a8ad573367ed382065e0fb95ec86e6c9cc8 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 8 Jun 2021 14:00:12 +0300 Subject: [PATCH 210/491] Fix test --- .../0_stateless/data_parquet/datapage_v2.snappy.parquet.columns | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/data_parquet/datapage_v2.snappy.parquet.columns b/tests/queries/0_stateless/data_parquet/datapage_v2.snappy.parquet.columns index d9e51028f22..c6bb5057cc2 100644 --- a/tests/queries/0_stateless/data_parquet/datapage_v2.snappy.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/datapage_v2.snappy.parquet.columns @@ -1 +1 @@ -`a` Nullable(String), `b` Nullable(Int32), `c` Nullable(Float64), `d` Nullable(UInt8), `e` Nullable(Int32) +`a` Nullable(String), `b` Array(Nullable(Int32)), `c` Nullable(Float64), `d` Nullable(UInt8), `e` Array(Nullable(Int32)) From 48cd8f2207c60a9f7498862d0d8c2d8285323881 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 11 Jun 2021 14:33:05 +0300 Subject: [PATCH 211/491] Fix build --- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 4 ++-- src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 856915acf3c..edf131cd49e 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -428,7 +428,7 @@ namespace DB default: throw Exception { - fmt::format("Unsupported {} type \"{}\" of an input column \"{}\".", format_name, arrow_column->type()->name(), column_name), + fmt::format(R"(Unsupported {} type "{}" of an input column "{}".)", format_name, arrow_column->type()->name(), column_name), ErrorCodes::UNKNOWN_TYPE }; } @@ -518,7 +518,7 @@ namespace DB } throw Exception { - fmt::format("The type \"{}\" of an input column \"{}\" is not supported for conversion from a {} data format.", arrow_type->name(), column_name, format_name), + fmt::format(R"(The type "{}" of an input column "{}" is not supported for conversion from a {} data format.)", arrow_type->name(), column_name, format_name), ErrorCodes::CANNOT_CONVERT_TYPE }; } diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 4dbb95b327a..cc487535e37 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -439,7 +439,7 @@ namespace DB { throw Exception { - fmt::format("Internal type \"{}\" of a column \"{}\" is not supported for conversion into a {} data format.", column_type_name, column_name, format_name), + fmt::format(R"(Internal type "{}" of a column "{}" is not supported for conversion into a {} data format.)", column_type_name, column_name, format_name), ErrorCodes::UNKNOWN_TYPE }; } @@ -580,7 +580,7 @@ namespace DB return arrow_type_it->second; } - throw Exception{fmt::format("The type \"{}\" of a column \"{}\" is not supported for conversion into a {} data format.", column_type->getName(), column_name, format_name), + throw Exception{fmt::format(R"(The type "{}" of a column "{}" is not supported for conversion into a {} data format.)", column_type->getName(), column_name, format_name), ErrorCodes::UNKNOWN_TYPE}; } From 30f3eacfe7a66c4570df7ecc584722e891e84ddb Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 16:32:54 +0300 Subject: [PATCH 212/491] First draft --- .../external-dicts-dict-sources.md | 19 +++++------ .../external-dicts-dict-sources.md | 32 +++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 619e4b8701b..69f7cfbc29c 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -113,15 +113,16 @@ Example of settings: cat /opt/dictionaries/os.tsv TabSeparated + false ``` Setting fields: -- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). -- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. -- `implicit_key` - The executable source file can return only values, and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. Default value is false. +- `command` — The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). +- `format` — The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. +- `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly — by the order of rows in the result. Default value is false. That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node. @@ -145,12 +146,12 @@ Example of settings: Setting fields: -- `command` – The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). -- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. -- `pool_size` - Size of pool. If 0 is specified as `pool_size` then there is no pool size restrictions. -- `command_termination_timeout` - Executable pool script, should contain main read-write loop. After dictionary is destroyed, pipe is closed, and executable file will have command_termination_timeout seconds to shutdown, before ClickHouse will send SIGTERM signal to child process. Specified in seconds. Default value is 10. Optional parameter. -- `max_command_execution_time` - Maximum executable script command execution time for processing block of data. Specified in seconds. Default value is 10. Optional parameter. -- `implicit_key` - The executable source file can return only values, and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. Default value is false. Optional parameter. +- `command` — The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). +- `format` — The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. +- `pool_size` — Size of pool. If 0 is specified as `pool_size` then there is no pool size restrictions. +- `command_termination_timeout` — Executable pool script, should contain main read-write loop. After dictionary is destroyed, pipe is closed, and executable file will have `command_termination_timeout` seconds to shutdown, before ClickHouse will send SIGTERM signal to child process. Specified in seconds. Default value is 10. Optional parameter. +- `max_command_execution_time` — Maximum executable script command execution time for processing block of data. Specified in seconds. Default value is 10. Optional parameter. +- `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. Default value is false. Optional parameter. That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index a0378251ece..d7658a939c6 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -69,6 +69,7 @@ SETTINGS(format_csv_allow_single_quotes = 0) - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) - [Redis](#dicts-external_dicts_dict_sources-redis) + - [Cassandra](#dicts-external_dicts_dict_sources-cassandra) - [PostgreSQL](#dicts-external_dicts_dict_sources-postgresql) ## Локальный файл {#dicts-external_dicts_dict_sources-local_file} @@ -112,6 +113,7 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) cat /opt/dictionaries/os.tsv TabSeparated + false ``` @@ -120,6 +122,36 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) - `command` — абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`). - `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». +- `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. + +Такой источник словаря можно настроить только через XML конфигурацию. Создание словарей с исполняемым источником через DDL отключено, иначе пользователи базы данных могли бы исполнять любой двоичный файл в узле ClickHouse. + +## Исполняемый пул {#dicts-external_dicts_dict_sources-executable_pool} + +Исполняемый пул позволяет загружать данные из пула процессов. Этот источник не работает с макетами словарей, которые требуют загрузки всех данных источника. Исполняемый пул работает, если тип размещения словаря `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct`. Исполняемый пул генерирует пул процессов с указанной командой и оставляет их активными, пока те не выйдут. Программа считывает данные из потока STDIN, пока он доступен, и выводит результат в поток STDOUT, кроме того возможно ожидание следующего блока данных из STDIN. ClickHouse не закрывает STDIN после обработки блока данных, но посылает следующую часть данных, когда это требуется. Исполняемый скрипт должен быть готов к такому способу передачи данных – он должен опросить STDIN и сбросить данные в STDOUT заранее. + +Пример настройки: + +``` xml + + + while read key; do printf "$key\tData for key $key\n"; done + TabSeparated + 10 + 10 + false + + +``` + +Поля настройки: + +- `command` — абсолютный путь к файлу или имя файла (если каталог программы записан в `PATH`). +- `format` — формат файла. Поддерживаются все форматы, описанные в “[Форматы](../../../interfaces/formats.md#formats)”. +- `pool_size` — размер пула. Если в поле `pool_size` указан 0, то размер пула не ограничен. +- `command_termination_timeout` — скрипт исполняемого пула, должен включать основной цикл чтения-записи. После уничтожения словаря канал закрывается. При этом исполняемый файл имеет `command_termination_timeout` секунд для завершения работы, прежде чем ClickHouse пошлет сигнал SIGTERM дочернему процессу. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. +- `max_command_execution_time` — максимальное количество времени для исполняемого скрипта на обработку блока данных. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. +- `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse. From d96478c9ff0240b0eaf23eca66172bee6544dc1d Mon Sep 17 00:00:00 2001 From: Kostiantyn Storozhuk Date: Tue, 15 Jun 2021 21:45:30 +0800 Subject: [PATCH 213/491] Integration test added and small fixes --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 10 +++++----- tests/integration/parallel.json | 2 ++ tests/integration/parallel_skip.json | 2 ++ .../materialize_with_ddl.py | 10 ++++++++++ .../test_materialize_mysql_database/test.py | 5 +++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index dd5dd311471..7d6b9fa37a2 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -35,7 +35,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int EMPTY_LIST_OF_COLUMNS_PASSED; - extern const int BAD_ARGUMENTS; } namespace MySQLInterpreter @@ -118,10 +117,10 @@ static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_defini return columns_name_and_type; } -static ColumnsDescription getColumnsDescription(const NamesAndTypesList & columns_name_and_type, const ASTExpressionList * columns_definition) +static ColumnsDescription getColumnsDescriptionFromList(const NamesAndTypesList & columns_name_and_type, const ASTExpressionList * columns_definition) { if (columns_name_and_type.size() != columns_definition->children.size()) - throw Exception("Columns of different size provided.", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Columns of different size provided.", ErrorCodes::LOGICAL_ERROR); ColumnsDescription columns_description; ColumnDescription column_description; @@ -142,7 +141,8 @@ static ColumnsDescription getColumnsDescription(const NamesAndTypesList & column column_description.name = column_name_and_type->name; column_description.type = column_name_and_type->type; - column_description.comment = std::move(comment); + if(!comment.empty()) + column_description.comment = std::move(comment); columns_description.add(column_description); } @@ -425,7 +425,7 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( NamesAndTypesList columns_name_and_type = getColumnsList(create_defines->columns); const auto & [primary_keys, unique_keys, keys, increment_columns] = getKeys(create_defines->columns, create_defines->indices, context, columns_name_and_type); - ColumnsDescription columns_description = getColumnsDescription(columns_name_and_type, create_defines->columns); + ColumnsDescription columns_description = getColumnsDescriptionFromList(columns_name_and_type, create_defines->columns); if (primary_keys.empty()) throw Exception("The " + backQuoteIfNeed(mysql_database) + "." + backQuoteIfNeed(create_query.table) diff --git a/tests/integration/parallel.json b/tests/integration/parallel.json index 4be0c9a77ae..f82e33138fc 100644 --- a/tests/integration/parallel.json +++ b/tests/integration/parallel.json @@ -175,6 +175,8 @@ "test_materialize_mysql_database/test.py::test_system_parts_table[clickhouse_node1]", "test_materialize_mysql_database/test.py::test_system_tables_table[clickhouse_node0]", "test_materialize_mysql_database/test.py::test_system_tables_table[clickhouse_node1]", + "test_materialize_mysql_database/test.py::test_materialize_with_column_comments[clickhouse_node0]", + "test_materialize_mysql_database/test.py::test_materialize_with_column_comments[clickhouse_node1]", "test_materialize_mysql_database/test.py::test_utf8mb4[clickhouse_node0]", "test_materialize_mysql_database/test.py::test_utf8mb4[clickhouse_node1]", "test_parts_delete_zookeeper/test.py::test_merge_doesnt_work_without_zookeeper", diff --git a/tests/integration/parallel_skip.json b/tests/integration/parallel_skip.json index cb6c5f735dc..7d124f4eac7 100644 --- a/tests/integration/parallel_skip.json +++ b/tests/integration/parallel_skip.json @@ -179,6 +179,8 @@ "test_materialize_mysql_database/test.py::test_system_parts_table[clickhouse_node1]", "test_materialize_mysql_database/test.py::test_system_tables_table[clickhouse_node0]", "test_materialize_mysql_database/test.py::test_system_tables_table[clickhouse_node1]", + "test_materialize_mysql_database/test.py::test_materialize_with_column_comments[clickhouse_node0]", + "test_materialize_mysql_database/test.py::test_materialize_with_column_comments[clickhouse_node1]", "test_materialize_mysql_database/test.py::test_utf8mb4[clickhouse_node0]", "test_materialize_mysql_database/test.py::test_utf8mb4[clickhouse_node1]", "test_parts_delete_zookeeper/test.py::test_merge_doesnt_work_without_zookeeper", diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 4091de45ac2..c5db90821e2 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -843,6 +843,16 @@ def system_tables_test(clickhouse_node, mysql_node, service_name): clickhouse_node.query("CREATE DATABASE system_tables_test ENGINE = MaterializeMySQL('{}:3306', 'system_tables_test', 'root', 'clickhouse')".format(service_name)) check_query(clickhouse_node, "SELECT partition_key, sorting_key, primary_key FROM system.tables WHERE database = 'system_tables_test' AND name = 'test'", "intDiv(id, 4294967)\tid\tid\n") +def materialize_with_column_comments_test(clickhouse_node, mysql_node, service_name): + mysql_node.query("DROP DATABASE IF EXISTS materialize_with_column_comments_test") + clickhouse_node.query("DROP DATABASE IF EXISTS materialize_with_column_comments_test") + mysql_node.query("CREATE DATABASE materialize_with_column_comments_test") + mysql_node.query("CREATE TABLE materialize_with_column_comments_test.test (id int NOT NULL PRIMARY KEY, value VARCHAR(255) COMMENT 'test comment') ENGINE=InnoDB") + clickhouse_node.query("CREATE DATABASE materialize_with_column_comments_test ENGINE = MaterializeMySQL('{}:3306', 'materialize_with_column_comments_test', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_column_comments_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\ttest comment\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + clickhouse_node.query("DROP DATABASE materialize_with_column_comments_test") + mysql_node.query("DROP DATABASE materialize_with_column_comments_test") + def move_to_prewhere_and_column_filtering(clickhouse_node, mysql_node, service_name): clickhouse_node.query("DROP DATABASE IF EXISTS cond_on_key_col") mysql_node.query("DROP DATABASE IF EXISTS cond_on_key_col") diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index e720bdfd5f5..e26500f07b3 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -218,6 +218,11 @@ def test_system_tables_table(started_cluster, started_mysql_8_0, started_mysql_5 materialize_with_ddl.system_tables_test(clickhouse_node, started_mysql_5_7, "mysql57") materialize_with_ddl.system_tables_test(clickhouse_node, started_mysql_8_0, "mysql80") +@pytest.mark.parametrize(('clickhouse_node'), [node_db_ordinary, node_db_ordinary]) +def test_materialize_with_column_comments(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): + materialize_with_ddl.materialize_with_column_comments_test(clickhouse_node, started_mysql_5_7, "mysql57") + materialize_with_ddl.materialize_with_column_comments_test(clickhouse_node, started_mysql_8_0, "mysql80") + @pytest.mark.parametrize(('clickhouse_node'), [node_disable_bytes_settings, node_disable_rows_settings]) def test_mysql_settings(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): From 481b87b37a9015cf0191e7143281a983f094dba1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 16:47:37 +0300 Subject: [PATCH 214/491] Remove some code from keyCondition. --- src/Storages/MergeTree/KeyCondition.cpp | 64 +++++++++---------------- src/Storages/MergeTree/KeyCondition.h | 11 ++--- 2 files changed, 26 insertions(+), 49 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index d624550d233..4907712b179 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -388,6 +388,14 @@ Block KeyCondition::getBlockWithConstants( return result; } +static NameSet getAllSubexpressionNames(const ExpressionActions & key_expr) +{ + NameSet names; + for (const auto & action : key_expr.getActions()) + names.insert(action.node->result_name); + + return names; +} KeyCondition::KeyCondition( const SelectQueryInfo & query_info, @@ -396,7 +404,11 @@ KeyCondition::KeyCondition( const ExpressionActionsPtr & key_expr_, bool single_point_, bool strict_) - : key_expr(key_expr_), prepared_sets(query_info.sets), single_point(single_point_), strict(strict_) + : key_expr(key_expr_) + , key_subexpr_names(getAllSubexpressionNames(*key_expr)) + , prepared_sets(query_info.sets) + , single_point(single_point_) + , strict(strict_) { for (size_t i = 0, size = key_column_names.size(); i < size; ++i) { @@ -589,45 +601,15 @@ void KeyCondition::traverseAST(const ASTPtr & node, ContextPtr context, Block & rpn.emplace_back(std::move(element)); } -bool KeyCondition::canConstantBeWrapped(const ASTPtr & node, const String & expr_name, String & result_expr_name) -{ - NameSet names; - for (const auto & action : key_expr->getActions()) - names.insert(action.node->result_name); - - /// sample_block from key_expr cannot contain modulo and moduloLegacy at the same time. - /// For partition key it is always moduloLegacy. - if (names.count(expr_name)) - { - result_expr_name = expr_name; - } - else - { - auto adjusted_ast = node->clone(); - KeyDescription::moduloToModuloLegacyRecursive(adjusted_ast); - String adjusted_expr_name = adjusted_ast->getColumnName(); - - if (!names.count(adjusted_expr_name)) - return false; - - result_expr_name = adjusted_expr_name; - } - - return true; -} - bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( - const ASTPtr & node [[maybe_unused]], - size_t & out_key_column_num [[maybe_unused]], - DataTypePtr & out_key_column_type [[maybe_unused]], - Field & out_value [[maybe_unused]], - DataTypePtr & out_type [[maybe_unused]]) + const ASTPtr & node, + size_t & out_key_column_num, + DataTypePtr & out_key_column_type, + Field & out_value, + DataTypePtr & out_type) { - - // Constant expr should use alias names if any - String passed_expr_name = node->getColumnNameWithoutAlias(); - String expr_name; - if (!canConstantBeWrapped(node, passed_expr_name, expr_name)) + String expr_name = node->getColumnNameWithoutAlias(); + if (key_subexpr_names.count(expr_name) == 0) return false; /// TODO Nullable index is not yet landed. @@ -729,10 +711,8 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( bool KeyCondition::canConstantBeWrappedByFunctions( const ASTPtr & ast, size_t & out_key_column_num, DataTypePtr & out_key_column_type, Field & out_value, DataTypePtr & out_type) { - // Constant expr should use alias names if any - String passed_expr_name = ast->getColumnNameWithoutAlias(); - String expr_name; - if (!canConstantBeWrapped(ast, passed_expr_name, expr_name)) + String expr_name = ast->getColumnNameWithoutAlias(); + if (key_subexpr_names.count(expr_name) == 0) return false; const auto & sample_block = key_expr->getSampleBlock(); diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index fc28d4a93c9..7e7b767b53b 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -419,12 +419,6 @@ private: bool canConstantBeWrappedByFunctions( const ASTPtr & ast, size_t & out_key_column_num, DataTypePtr & out_key_column_type, Field & out_value, DataTypePtr & out_type); - /// Check if ASTPtr node, passed to canConstantBeWrappedBy*, can be used by them for further checks. - /// Always call this method at start of other methods, which require key comparison, because it also checks if adjusted - /// key expression can also be used (with substitution from modulo to moduloLegacy). This is needed because partition key - /// is always modified, when passed into keyCondition, - with recursive substitution from modulo to moduloLegacy. - bool canConstantBeWrapped(const ASTPtr & node, const String & expr_name, String & result_expr_name); - /// If it's possible to make an RPNElement /// that will filter values (possibly tuples) by the content of 'prepared_set', /// do it and return true. @@ -461,7 +455,10 @@ private: RPN rpn; ColumnIndices key_columns; - ExpressionActionsPtr key_expr; + /// Expression which is used for key condition. + const ExpressionActionsPtr key_expr; + /// All intermediate columns are used to calculate key_expr. + const NameSet key_subexpr_names; PreparedSets prepared_sets; // If true, always allow key_expr to be wrapped by function From 81b1ef1b0c0dac1221c09c199d33b20bb624be24 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 16:58:42 +0300 Subject: [PATCH 215/491] Small fixes --- .../external-dictionaries/external-dicts-dict-sources.md | 4 ++-- .../external-dictionaries/external-dicts-dict-sources.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 69f7cfbc29c..bfb710b0ded 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -128,7 +128,7 @@ That dictionary source can be configured only via XML configuration. Creating di ## Executable Pool {#dicts-external_dicts_dict_sources-executable_pool} -Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary is stored using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. Executable pool will spawn pool of processes with specified command and keep them running until they exit. The program should read data from STDIN while it is available and output result to STDOUT, and it can wait for next block of data on stdin. ClickHouse will not close STDIN after processing a block of data but will pipe another chunk of data when needed. The executable script should be ready for this way of data processing - it should poll STDIN and flush data to STDOUT early. +Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary is stored using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. Executable pool will spawn pool of processes with specified command and keep them running until they exit. The program should read data from STDIN while it is available and output result to STDOUT, and it can wait for next block of data on stdin. ClickHouse will not close STDIN after processing a block of data but will pipe another chunk of data when needed. The executable script should be ready for this way of data processing — it should poll STDIN and flush data to STDOUT early. Example of settings: @@ -151,7 +151,7 @@ Setting fields: - `pool_size` — Size of pool. If 0 is specified as `pool_size` then there is no pool size restrictions. - `command_termination_timeout` — Executable pool script, should contain main read-write loop. After dictionary is destroyed, pipe is closed, and executable file will have `command_termination_timeout` seconds to shutdown, before ClickHouse will send SIGTERM signal to child process. Specified in seconds. Default value is 10. Optional parameter. - `max_command_execution_time` — Maximum executable script command execution time for processing block of data. Specified in seconds. Default value is 10. Optional parameter. -- `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly - by the order of rows in the result. Default value is false. Optional parameter. +- `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly — by the order of rows in the result. Default value is false. Optional parameter. That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index d7658a939c6..700ba689aa5 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -124,11 +124,11 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) - `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». - `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. -Такой источник словаря можно настроить только через XML конфигурацию. Создание словарей с исполняемым источником через DDL отключено, иначе пользователи базы данных могли бы исполнять любой двоичный файл в узле ClickHouse. +Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse. ## Исполняемый пул {#dicts-external_dicts_dict_sources-executable_pool} -Исполняемый пул позволяет загружать данные из пула процессов. Этот источник не работает с макетами словарей, которые требуют загрузки всех данных источника. Исполняемый пул работает, если тип размещения словаря `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct`. Исполняемый пул генерирует пул процессов с указанной командой и оставляет их активными, пока те не выйдут. Программа считывает данные из потока STDIN, пока он доступен, и выводит результат в поток STDOUT, кроме того возможно ожидание следующего блока данных из STDIN. ClickHouse не закрывает STDIN после обработки блока данных, но посылает следующую часть данных, когда это требуется. Исполняемый скрипт должен быть готов к такому способу передачи данных – он должен опросить STDIN и сбросить данные в STDOUT заранее. +Исполняемый пул позволяет загружать данные из пула процессов. Этот источник не работает с макетами словарей, которые требуют загрузки всех данных источника. Исполняемый пул работает, если тип размещения словаря `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct`. Исполняемый пул генерирует пул процессов с указанной командой и оставляет их активными, пока те не выйдут. Программа считывает данные из потока STDIN, пока он доступен, и выводит результат в поток STDOUT, кроме того возможно ожидание следующего блока данных из STDIN. ClickHouse не закрывает STDIN после обработки блока данных, но посылает следующую часть данных, когда это требуется. Исполняемый скрипт должен быть готов к такому способу передачи данных — он должен опросить STDIN и сбросить данные в STDOUT заранее. Пример настройки: From 70b8758bde69f723dd48eeca90f7df487b9e480b Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 15 Jun 2021 17:05:43 +0300 Subject: [PATCH 216/491] Apply suggestions from code review Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/data-types/array.md | 4 +--- docs/en/sql-reference/data-types/tuple.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index ac9cbdd9a6e..76ed8beb5eb 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -81,11 +81,9 @@ It is possible to find the size of an array by using the `size0` subcolumn witho Query: ```sql -CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple(); - CREATE TABLE t_arr (`arr` Array(Array(Array(UInt32)))) ENGINE = MergeTree ORDER BY tuple(); -insert into t_arr values ([[[12, 13, 0, 1],[12]]]); +INSERT INTO t_arr VALUES ([[[12, 13, 0, 1],[12]]]); SELECT arr.size0, arr.size1, arr.size2 FROM t_arr; ``` diff --git a/docs/en/sql-reference/data-types/tuple.md b/docs/en/sql-reference/data-types/tuple.md index 9092643ed42..dea5e10365f 100644 --- a/docs/en/sql-reference/data-types/tuple.md +++ b/docs/en/sql-reference/data-types/tuple.md @@ -47,7 +47,7 @@ SELECT tuple(1, NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` -## Addressing Tuple Element {#addressing-tuple-element} +## Addressing Tuple Elements {#addressing-tuple-elements} It is possible to read elements of named tuples using indexes and names: From b182d87d9c84ffe88b3f9e0959fd314c6dd58aeb Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 15 Jun 2021 17:33:46 +0300 Subject: [PATCH 217/491] Add settings for HTTP header limitations --- programs/library-bridge/Handlers.cpp | 2 +- programs/odbc-bridge/ColumnInfoHandler.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/odbc-bridge/SchemaAllowedHandler.cpp | 2 +- src/Core/Settings.h | 3 ++ src/Server/HTTP/HTMLForm.cpp | 36 ++++++++++--------- src/Server/HTTP/HTMLForm.h | 26 ++++++-------- src/Server/HTTP/HTTPServerRequest.cpp | 5 ++- src/Server/HTTP/HTTPServerRequest.h | 5 +-- src/Server/HTTP/ReadHeaders.h | 6 +--- src/Server/HTTPHandler.cpp | 2 +- src/Server/InterserverIOHTTPHandler.cpp | 2 +- src/Server/InterserverIOHTTPHandler.h | 2 +- src/Server/ReplicasStatusHandler.cpp | 13 +++---- src/Server/ReplicasStatusHandler.h | 5 +-- 16 files changed, 53 insertions(+), 62 deletions(-) diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index 6a1bfbbccb7..bc955705556 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -51,7 +51,7 @@ namespace void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { LOG_TRACE(log, "Request URI: {}", request.getURI()); - HTMLForm params(request); + HTMLForm params(getContext(), request); if (!params.has("method")) { diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index f4f575bb33d..c968ff12f5b 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -69,7 +69,7 @@ namespace void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(request, request.getStream()); + HTMLForm params(getContext(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index 124a5c420f8..2919519f990 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -21,7 +21,7 @@ namespace DB { void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(request, request.getStream()); + HTMLForm params(getContext(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index ffa636e8b49..d8aa617c359 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -50,7 +50,7 @@ void ODBCHandler::processError(HTTPServerResponse & response, const std::string void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(request); + HTMLForm params(getContext(), request); LOG_TRACE(log, "Request URI: {}", request.getURI()); if (mode == "read") diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index 3a20148780d..5bf996ec2b4 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -28,7 +28,7 @@ namespace void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(request, request.getStream()); + HTMLForm params(getContext(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2aed174c088..fca593be020 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -234,6 +234,9 @@ class IColumn; M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \ M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \ M(UInt64, http_max_uri_size, 1048576, "Maximum URI length of HTTP request", 0) \ + M(UInt64, http_max_fields_number, 1000000, "Maximum number of fields in HTTP header", 0) \ + M(UInt64, http_max_field_name_size, 1024, "Maximum length of field name in HTTP header", 0) \ + M(UInt64, http_max_field_value_size, 8192, "Maximum length of field value in HTTP header", 0) \ M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \ M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ diff --git a/src/Server/HTTP/HTMLForm.cpp b/src/Server/HTTP/HTMLForm.cpp index 9e0f74dcc33..67f97153861 100644 --- a/src/Server/HTTP/HTMLForm.cpp +++ b/src/Server/HTTP/HTMLForm.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -35,38 +36,41 @@ const std::string HTMLForm::ENCODING_MULTIPART = "multipart/form-data"; const int HTMLForm::UNKNOWN_CONTENT_LENGTH = -1; -HTMLForm::HTMLForm() : field_limit(DFL_FIELD_LIMIT), value_length_limit(DFL_MAX_VALUE_LENGTH), encoding(ENCODING_URL) +HTMLForm::HTMLForm(ContextPtr context) + : max_fields_number(context->getSettingsRef().http_max_fields_number) + , max_field_name_size(context->getSettingsRef().http_max_field_name_size) + , max_field_value_size(context->getSettingsRef().http_max_field_value_size) + , encoding(ENCODING_URL) { } -HTMLForm::HTMLForm(const std::string & encoding_) - : field_limit(DFL_FIELD_LIMIT), value_length_limit(DFL_MAX_VALUE_LENGTH), encoding(encoding_) +HTMLForm::HTMLForm(ContextPtr context, const std::string & encoding_) : HTMLForm(context) { + encoding = encoding_; } -HTMLForm::HTMLForm(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler) - : field_limit(DFL_FIELD_LIMIT), value_length_limit(DFL_MAX_VALUE_LENGTH) +HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler) + : HTMLForm(context) { load(request, requestBody, handler); } -HTMLForm::HTMLForm(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody) - : field_limit(DFL_FIELD_LIMIT), value_length_limit(DFL_MAX_VALUE_LENGTH) +HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody) : HTMLForm(context) { load(request, requestBody); } -HTMLForm::HTMLForm(const Poco::Net::HTTPRequest & request) : HTMLForm(Poco::URI(request.getURI())) +HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request) : HTMLForm(context, Poco::URI(request.getURI())) { } -HTMLForm::HTMLForm(const Poco::URI & uri) : field_limit(DFL_FIELD_LIMIT), value_length_limit(DFL_MAX_VALUE_LENGTH) +HTMLForm::HTMLForm(ContextPtr context, const Poco::URI & uri) : HTMLForm(context) { - ReadBufferFromString istr(uri.getRawQuery()); // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ReadBufferFromString istr(uri.getRawQuery()); // STYLE_CHECK_ALLOW_STD_STRING_STREAM readQuery(istr); } @@ -123,7 +127,7 @@ void HTMLForm::readQuery(ReadBuffer & in) while (true) { - if (field_limit > 0 && fields == field_limit) + if (max_fields_number > 0 && fields == max_fields_number) throw Poco::Net::HTMLFormException("Too many form fields"); std::string name; @@ -133,7 +137,7 @@ void HTMLForm::readQuery(ReadBuffer & in) { if (ch == '+') ch = ' '; - if (name.size() < MAX_NAME_LENGTH) + if (name.size() < max_field_name_size) name += ch; else throw Poco::Net::HTMLFormException("Field name too long"); @@ -145,7 +149,7 @@ void HTMLForm::readQuery(ReadBuffer & in) { if (ch == '+') ch = ' '; - if (value.size() < value_length_limit) + if (value.size() < max_field_value_size) value += ch; else throw Poco::Net::HTMLFormException("Field value too long"); @@ -185,11 +189,11 @@ void HTMLForm::readMultipart(ReadBuffer & in_, PartHandler & handler) /// Read each part until next boundary (or last boundary) while (!in.eof()) { - if (field_limit && fields > field_limit) + if (max_fields_number && fields > max_fields_number) throw Poco::Net::HTMLFormException("Too many form fields"); Poco::Net::MessageHeader header; - readHeaders(header, in); + readHeaders(header, in, max_fields_number, max_field_name_size, max_field_value_size); skipToNextLineOrEOF(in); NameValueCollection params; @@ -209,7 +213,7 @@ void HTMLForm::readMultipart(ReadBuffer & in_, PartHandler & handler) while (in.read(ch)) { - if (value.size() > value_length_limit) + if (value.size() > max_field_value_size) throw Poco::Net::HTMLFormException("Field value too long"); value += ch; } diff --git a/src/Server/HTTP/HTMLForm.h b/src/Server/HTTP/HTMLForm.h index ca6bb9048f1..7935f7b53f1 100644 --- a/src/Server/HTTP/HTMLForm.h +++ b/src/Server/HTTP/HTMLForm.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -19,31 +20,31 @@ public: enum Options { - OPT_USE_CONTENT_LENGTH = 0x01 // don't use Chunked Transfer-Encoding for multipart requests. + OPT_USE_CONTENT_LENGTH = 0x01, /// don't use Chunked Transfer-Encoding for multipart requests. }; /// Creates an empty HTMLForm and sets the /// encoding to "application/x-www-form-urlencoded". - HTMLForm(); + explicit HTMLForm(ContextPtr context); /// Creates an empty HTMLForm that uses the given encoding. /// Encoding must be either "application/x-www-form-urlencoded" (which is the default) or "multipart/form-data". - explicit HTMLForm(const std::string & encoding); + explicit HTMLForm(ContextPtr context, const std::string & encoding); /// Creates a HTMLForm from the given HTTP request. /// Uploaded files are passed to the given PartHandler. - HTMLForm(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler); + HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler); /// Creates a HTMLForm from the given HTTP request. /// Uploaded files are silently discarded. - HTMLForm(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody); + HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody); /// Creates a HTMLForm from the given HTTP request. /// The request must be a GET request and the form data must be in the query string (URL encoded). /// For POST requests, you must use one of the constructors taking an additional input stream for the request body. - explicit HTMLForm(const Poco::Net::HTTPRequest & request); + explicit HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request); - explicit HTMLForm(const Poco::URI & uri); + explicit HTMLForm(ContextPtr context, const Poco::URI & uri); template T getParsed(const std::string & key, T default_value) @@ -76,13 +77,6 @@ private: /// This buffer provides data line by line to check for boundary line in a convenient way. class MultipartReadBuffer; - enum Limits - { - DFL_FIELD_LIMIT = 100, - MAX_NAME_LENGTH = 1024, - DFL_MAX_VALUE_LENGTH = 256 * 1024 - }; - struct Part { std::string name; @@ -91,8 +85,8 @@ private: using PartVec = std::vector; - size_t field_limit; - size_t value_length_limit; + const size_t max_fields_number, max_field_name_size, max_field_value_size; + std::string encoding; std::string boundary; PartVec parts; diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index 69dc8d4dbda..d4359ee0355 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -17,6 +17,9 @@ namespace DB { HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) : max_uri_size(context->getSettingsRef().http_max_uri_size) + , max_fields_number(context->getSettingsRef().http_max_fields_number) + , max_field_name_size(context->getSettingsRef().http_max_field_name_size) + , max_field_value_size(context->getSettingsRef().http_max_field_value_size) { response.attachRequest(this); @@ -110,7 +113,7 @@ void HTTPServerRequest::readRequest(ReadBuffer & in) skipToNextLineOrEOF(in); - readHeaders(*this, in); + readHeaders(*this, in, max_fields_number, max_field_name_size, max_field_value_size); skipToNextLineOrEOF(in); diff --git a/src/Server/HTTP/HTTPServerRequest.h b/src/Server/HTTP/HTTPServerRequest.h index a560f907cf0..c6e8d2dd728 100644 --- a/src/Server/HTTP/HTTPServerRequest.h +++ b/src/Server/HTTP/HTTPServerRequest.h @@ -40,14 +40,11 @@ private: /// Limits for basic sanity checks when reading a header enum Limits { - MAX_NAME_LENGTH = 256, - MAX_VALUE_LENGTH = 8192, MAX_METHOD_LENGTH = 32, MAX_VERSION_LENGTH = 8, - MAX_FIELDS_NUMBER = 100, }; - const size_t max_uri_size; + const size_t max_uri_size, max_fields_number, max_field_name_size, max_field_value_size; std::unique_ptr stream; Poco::Net::SocketImpl * socket; diff --git a/src/Server/HTTP/ReadHeaders.h b/src/Server/HTTP/ReadHeaders.h index e94cddcf489..1b0e627f779 100644 --- a/src/Server/HTTP/ReadHeaders.h +++ b/src/Server/HTTP/ReadHeaders.h @@ -8,10 +8,6 @@ namespace DB class ReadBuffer; void readHeaders( - Poco::Net::MessageHeader & headers, - ReadBuffer & in, - size_t max_fields_number = 100, - size_t max_name_length = 256, - size_t max_value_length = 8192); + Poco::Net::MessageHeader & headers, ReadBuffer & in, size_t max_fields_number, size_t max_name_length, size_t max_value_length); } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 6eab6cdda5e..28105de7111 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -895,7 +895,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (request.getVersion() == HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); - HTMLForm params(request); + HTMLForm params(request_context, request); with_stacktrace = params.getParsed("stacktrace", false); /// FIXME: maybe this check is already unnecessary. diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 64af8860b23..5329d4f77fd 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -50,7 +50,7 @@ std::pair InterserverIOHTTPHandler::checkAuthentication(HTTPServer void InterserverIOHTTPHandler::processQuery(HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output) { - HTMLForm params(request); + HTMLForm params(server.context(), request); LOG_TRACE(log, "Request URI: {}", request.getURI()); diff --git a/src/Server/InterserverIOHTTPHandler.h b/src/Server/InterserverIOHTTPHandler.h index c0d776115e1..da5b286b9e5 100644 --- a/src/Server/InterserverIOHTTPHandler.h +++ b/src/Server/InterserverIOHTTPHandler.h @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include #include diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 86295cc5170..7cb47b85cda 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -18,23 +18,20 @@ namespace DB { - -ReplicasStatusHandler::ReplicasStatusHandler(IServer & server) - : context(server.context()) +ReplicasStatusHandler::ReplicasStatusHandler(IServer & server) : WithContext(server.context()) { } - void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try { - HTMLForm params(request); + HTMLForm params(getContext(), request); /// Even if lag is small, output detailed information about the lag. bool verbose = params.get("verbose", "") == "1"; - const MergeTreeSettings & settings = context->getReplicatedMergeTreeSettings(); + const MergeTreeSettings & settings = getContext()->getReplicatedMergeTreeSettings(); bool ok = true; WriteBufferFromOwnString message; @@ -48,7 +45,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe if (!db.second->canContainMergeTreeTables()) continue; - for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) + for (auto iterator = db.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) { const auto & table = iterator->table(); if (!table) @@ -73,7 +70,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe } } - const auto & config = context->getConfigRef(); + const auto & config = getContext()->getConfigRef(); setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); if (!ok) diff --git a/src/Server/ReplicasStatusHandler.h b/src/Server/ReplicasStatusHandler.h index eda0b15ed6f..1a5388aa2ab 100644 --- a/src/Server/ReplicasStatusHandler.h +++ b/src/Server/ReplicasStatusHandler.h @@ -9,11 +9,8 @@ class Context; class IServer; /// Replies "Ok.\n" if all replicas on this server don't lag too much. Otherwise output lag information. -class ReplicasStatusHandler : public HTTPRequestHandler +class ReplicasStatusHandler : public HTTPRequestHandler, WithContext { -private: - ContextPtr context; - public: explicit ReplicasStatusHandler(IServer & server_); From c50eb7881971adb081f22068105626ca4b1224fb Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 17:38:08 +0300 Subject: [PATCH 218/491] Updated EN version --- docs/en/sql-reference/data-types/array.md | 2 -- docs/en/sql-reference/data-types/nullable.md | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 76ed8beb5eb..741bbd4e153 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -95,5 +95,3 @@ Result: │ 1 │ [2] │ [[4,1]] │ └───────────┴───────────┴───────────┘ ``` - -[Original article](https://clickhouse.tech/docs/en/data_types/array/) diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index a526a99d9a8..3c7b53d1e56 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -20,7 +20,11 @@ To store `Nullable` type values in a table column, ClickHouse uses a separate fi !!! info "Note" Using `Nullable` almost always negatively affects performance, keep this in mind when designing your databases. -It is possible to find `NULL` values in a column by using `null` subcolumn without reading the whole column. +## Finding NULL {#finding-null} + +It is possible to find `NULL` values in a column by using `null` subcolumn without reading the whole column. It returns `1` if the corresponding value is `NULL` and `0` otherwise. + +Query: ``` sql CREATE TABLE nullable ( `n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); From 3ee64dc629fc8d7458c3968ad61786fceb6d0a14 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 17:39:31 +0300 Subject: [PATCH 219/491] Added translation --- docs/ru/sql-reference/data-types/array.md | 25 ++++++++++++++--- .../nested-data-structures/nested.md | 2 +- docs/ru/sql-reference/data-types/nullable.md | 25 +++++++++++++++++ docs/ru/sql-reference/data-types/tuple.md | 27 +++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index 30952d6e126..345a294e9a2 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -5,9 +5,7 @@ toc_title: Array(T) # Array(T) {#data-type-array} -Массив из элементов типа `T`. - -`T` может любым, в том числе, массивом. Таким образом поддержаны многомерные массивы. +Массив из элементов типа `T`. `T` может любым, в том числе, массивом. Таким образом поддержаны многомерные массивы. ## Создание массива {#sozdanie-massiva} @@ -76,3 +74,24 @@ Received exception from server (version 1.1.54388): Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. ``` +## Размер массива {#array-size} + +Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец size`n-1`, где `n` — требуемое измерение. + +Запрос: + +```sql +CREATE TABLE t_arr (`arr` Array(Array(Array(UInt32)))) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_arr VALUES ([[[12, 13, 0, 1],[12]]]); + +SELECT arr.size0, arr.size1, arr.size2 FROM t_arr; +``` + +Результат: + +``` text +┌─arr.size0─┬─arr.size1─┬─arr.size2─┐ +│ 1 │ [2] │ [[4,1]] │ +└───────────┴───────────┴───────────┘ +``` diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md index 199d141a191..e192db755fc 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md @@ -29,7 +29,7 @@ CREATE TABLE test.visits В этом примере объявлена вложенная структура данных `Goals`, содержащая данные о достижении целей. Каждой строке таблицы visits может соответствовать от нуля до произвольного количества достижений целей. -Поддерживается только один уровень вложенности. Столбцы вложенных структур, содержащие массивы, эквивалентны многомерным массивам, поэтому их поддержка ограничена (не поддерживается хранение таких столбцов в таблицах с движком семейства MergeTree). +Поддерживаются любые уровни вложенности. Столбцы вложенных структур, содержащие массивы, эквивалентны многомерным массивам, поэтому их поддержка ограничена (не поддерживается хранение таких столбцов в таблицах с движком семейства MergeTree). В большинстве случаев, при работе с вложенной структурой данных, указываются отдельные её столбцы. Для этого, имена столбцов указываются через точку. Эти столбцы представляют собой массивы соответствующих типов. Все столбцы-массивы одной вложенной структуры данных имеют одинаковые длины. diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index 3f33c4b2540..bd60f143d51 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -27,6 +27,31 @@ toc_title: Nullable !!! info "Info" Почти всегда использование `Nullable` снижает производительность, учитывайте это при проектировании своих баз. +## Поиск NULL {#finding-null} + +Найти значения `NULL` в столбце можно с помощью подстобца `null`, при этом чтение всего столбца не происходит. Подстолбец возвращает `1`, если соответственное значение является `NULL`, и `0` в противоположном случае. + +Запрос: + +``` sql +CREATE TABLE nullable ( `n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); + +SELECT n.null FROM nullable; +``` + +Результат: + +``` text +┌─n.null─┐ +│ 0 │ +│ 1 │ +│ 0 │ +│ 1 │ +└────────┘ +``` + ## Пример использования {#primer-ispolzovaniia} ``` sql diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index 702b5962f7b..fa46831240a 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -47,3 +47,30 @@ SELECT tuple(1,NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` +## Адрессация элементов кортежа {#addressing-tuple-elements} + +Можно считывать элементы кортежей с помощью индексов и имен: + +``` sql +CREATE TABLE named_tuples (`a` Tuple(s String, i Int64)) ENGINE = Memory; + +INSERT INTO named_tuples VALUES (('y', 10)), (('x',-10)); + +SELECT a.s FROM named_tuples; + +SELECT a.2 FROM named_tuples; +``` + +Результат: + +``` text +┌─a.s─┐ +│ y │ +│ x │ +└─────┘ + +┌─tupleElement(a, 2)─┐ +│ 10 │ +│ -10 │ +└────────────────────┘ +``` From 69f571d0ce139d2ed79eb4f0e200404f6086f96d Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 17:46:00 +0300 Subject: [PATCH 220/491] Typo --- docs/ru/sql-reference/data-types/tuple.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index fa46831240a..babff343a29 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -47,7 +47,7 @@ SELECT tuple(1,NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` -## Адрессация элементов кортежа {#addressing-tuple-elements} +## Адресация элементов кортежа {#addressing-tuple-elements} Можно считывать элементы кортежей с помощью индексов и имен: From 2b7547cb13946c58b4478961203ec912d6a23934 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 17:52:23 +0300 Subject: [PATCH 221/491] Small update --- docs/en/sql-reference/data-types/array.md | 2 +- docs/ru/sql-reference/data-types/array.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 741bbd4e153..a76fb9e2e8c 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -76,7 +76,7 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception ## Array Size {#array-size} -It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use size`n-1`, where `n` is the wanted dimension. +It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use `sizen-1`, where `n` is the wanted dimension. Query: diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index 345a294e9a2..ec45abac7fb 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -76,7 +76,7 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception ## Размер массива {#array-size} -Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец size`n-1`, где `n` — требуемое измерение. +Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец `sizen-1`, где `n` — требуемое измерение. Запрос: From 82919778b5847bc18abdc1a2f88b7bbf559ef468 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 17:53:58 +0300 Subject: [PATCH 222/491] Update test. --- tests/queries/0_stateless/01540_verbatim_partition_pruning.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql index 2d227856be4..bd92290056a 100644 --- a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql +++ b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql @@ -14,7 +14,7 @@ insert into xy values (0, 2), (2, 3), (8, 4), (9, 5); SET max_rows_to_read = 2; -select * from xy where intHash64(x) % 2 = intHash64(2) % 2; +select * from xy where moduloLegacy(intHash64(x), 2) = moduloLegacy(intHash64(2), 2); -- Equality is another special operator that can be treated as an always monotonic indicator for deterministic functions. -- minmax index is not enough. From 2f596fba050652b0a2feda4dd0a03a6dfdc78173 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 17:57:49 +0300 Subject: [PATCH 223/491] Minor improvements --- docs/en/sql-reference/data-types/nullable.md | 2 +- docs/ru/sql-reference/data-types/nullable.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 3c7b53d1e56..7b43beb65f4 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -27,7 +27,7 @@ It is possible to find `NULL` values in a column by using `null` subcolumn witho Query: ``` sql -CREATE TABLE nullable ( `n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index bd60f143d51..f3581108dd4 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -29,12 +29,12 @@ toc_title: Nullable ## Поиск NULL {#finding-null} -Найти значения `NULL` в столбце можно с помощью подстобца `null`, при этом чтение всего столбца не происходит. Подстолбец возвращает `1`, если соответственное значение является `NULL`, и `0` в противоположном случае. +Найти в столбце значения `NULL` можно с помощью подстобца `null`, при этом чтение всего столбца не происходит. Подстолбец возвращает `1`, если соответственное значение равно `NULL`, и `0` в противоположном случае. Запрос: ``` sql -CREATE TABLE nullable ( `n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO nullable VALUES (1) (NULL) (2) (NULL); From b8c20f201baefbe3839f42c73a0eda9eb57897a1 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 15 Jun 2021 18:11:11 +0300 Subject: [PATCH 224/491] Small update --- .../external-dictionaries/external-dicts-dict-sources.md | 1 + .../external-dictionaries/external-dicts-dict-sources.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index bfb710b0ded..0077f618c04 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -62,6 +62,7 @@ Types of sources (`source_type`): - [Local file](#dicts-external_dicts_dict_sources-local_file) - [Executable file](#dicts-external_dicts_dict_sources-executable) - [HTTP(s)](#dicts-external_dicts_dict_sources-http) +- [Executable Pool](#dicts-external_dicts_dict_sources-executable_pool) - DBMS - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 700ba689aa5..67a8de29e2a 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -62,6 +62,7 @@ SETTINGS(format_csv_allow_single_quotes = 0) - [Локальный файл](#dicts-external_dicts_dict_sources-local_file) - [Исполняемый файл](#dicts-external_dicts_dict_sources-executable) - [HTTP(s)](#dicts-external_dicts_dict_sources-http) +- [Исполняемый пул](#dicts-external_dicts_dict_sources-executable_pool) - СУБД: - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) From 74849bbe0c9aba8e7958b28d49702b3653733c5e Mon Sep 17 00:00:00 2001 From: Storozhuk Kostiantyn <56565543+sand6255@users.noreply.github.com> Date: Tue, 15 Jun 2021 18:21:02 +0300 Subject: [PATCH 225/491] Update InterpretersMySQLDDLQuery.cpp Added space after if --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 7d6b9fa37a2..c0815ded0bc 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -141,7 +141,7 @@ static ColumnsDescription getColumnsDescriptionFromList(const NamesAndTypesList column_description.name = column_name_and_type->name; column_description.type = column_name_and_type->type; - if(!comment.empty()) + if (!comment.empty()) column_description.comment = std::move(comment); columns_description.add(column_description); } From 80a13c489b3af70b7083cbee0febf0df36ae0388 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 18:21:31 +0300 Subject: [PATCH 226/491] Revert back moduloLegacy check for canConstantBeWrappedByFunctions. --- src/Storages/MergeTree/KeyCondition.cpp | 16 +++++++++++++++- .../01540_verbatim_partition_pruning.sql | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 4907712b179..efa238ccbfd 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -712,8 +712,22 @@ bool KeyCondition::canConstantBeWrappedByFunctions( const ASTPtr & ast, size_t & out_key_column_num, DataTypePtr & out_key_column_type, Field & out_value, DataTypePtr & out_type) { String expr_name = ast->getColumnNameWithoutAlias(); + if (key_subexpr_names.count(expr_name) == 0) - return false; + { + /// Let's check another one case. + /// If our storage was created with moduloLegacy in partition key, + /// We can assume that `modulo(...) = const` is the same as `moduloLegacy(...) = const`. + /// Replace modulo to moduloLegacy in AST and check if we also have such a column. + /// + /// Note: for negative values, we can filter more partitions then needed. + auto adjusted_ast = ast->clone(); + KeyDescription::moduloToModuloLegacyRecursive(adjusted_ast); + expr_name = adjusted_ast->getColumnName(); + + if (key_subexpr_names.count(expr_name) == 0) + return false; + } const auto & sample_block = key_expr->getSampleBlock(); diff --git a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql index bd92290056a..2d227856be4 100644 --- a/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql +++ b/tests/queries/0_stateless/01540_verbatim_partition_pruning.sql @@ -14,7 +14,7 @@ insert into xy values (0, 2), (2, 3), (8, 4), (9, 5); SET max_rows_to_read = 2; -select * from xy where moduloLegacy(intHash64(x), 2) = moduloLegacy(intHash64(2), 2); +select * from xy where intHash64(x) % 2 = intHash64(2) % 2; -- Equality is another special operator that can be treated as an always monotonic indicator for deterministic functions. -- minmax index is not enough. From fb1c3d0360ee1c2639d9210b9995f410b3447e40 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 15 Jun 2021 18:47:50 +0300 Subject: [PATCH 227/491] Update libpqxx --- contrib/libpqxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libpqxx b/contrib/libpqxx index 78f4488db02..ec49964f89e 160000 --- a/contrib/libpqxx +++ b/contrib/libpqxx @@ -1 +1 @@ -Subproject commit 78f4488db0266d15ab0af5a8a3791065916e1cc8 +Subproject commit ec49964f89eddc4e117fb93156f8ae6c75d33a1f From d94b9c592d2577c5e036d8bcd4e4785d32a19f78 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 18:51:06 +0300 Subject: [PATCH 228/491] Update in.cpp --- src/Functions/in.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 3448e09b990..7cd9f64004d 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -131,7 +131,6 @@ public: columns_of_key_columns.insert(left_arg); /// Replace single LowCardinality column to it's dictionary if possible. - /// It is good enough optimization, but it does not use LowCardinality cache. ColumnPtr lc_indexes = nullptr; if (columns_of_key_columns.columns() == 1) { From f44bdc0b756cbe0b947c5b307514703828abf4a3 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 15 Jun 2021 18:09:10 +0200 Subject: [PATCH 229/491] Changes to rounding and naming. Enabling the extended datatypes tests. --- .../snapshots/common.py.tests.snapshot | 1230 ++++++++--------- .../tests/arithmetic.py | 8 +- .../tests/array_tuple_map.py | 24 +- .../tests/comparison.py | 5 +- .../tests/conversion.py | 20 +- .../tests/logical.py | 1 + .../tests/mathematical.py | 12 +- .../tests/null.py | 7 +- tests/testflows/regression.py | 2 +- 9 files changed, 663 insertions(+), 646 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot b/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot index 6e3848b9e68..d0b7b3423d8 100644 --- a/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot +++ b/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot @@ -1,160 +1,160 @@ I_check_plus_with_Int128_max_and_min_value = r""" -plus(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) plus(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(plus(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(plus(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) -170141183460469231731687303715884105728 -170141183460469231731687303715884105727 """ I_check_plus_with_Int256_max_and_min_value = r""" -plus(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) plus(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(plus(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(plus(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) -57896044618658097711785492504343953926634992332820282019728792003956564819968 -57896044618658097711785492504343953926634992332820282019728792003956564819967 """ I_check_plus_with_UInt128_max_and_min_value = r""" -plus(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) plus(toUInt128(\'0\'), toUInt128(1)) +round(plus(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(plus(toUInt128(\'0\'), toUInt128(1)), 7) 0 1 """ I_check_plus_with_UInt256_max_and_min_value = r""" -plus(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) plus(toUInt256(\'0\'), toUInt256(1)) +round(plus(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(plus(toUInt256(\'0\'), toUInt256(1)), 7) 0 1 """ I_check_minus_with_Int128_max_and_min_value = r""" -minus(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) minus(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(minus(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(minus(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 170141183460469231731687303715884105726 170141183460469231731687303715884105727 """ I_check_minus_with_Int256_max_and_min_value = r""" -minus(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) minus(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(minus(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(minus(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 57896044618658097711785492504343953926634992332820282019728792003956564819966 57896044618658097711785492504343953926634992332820282019728792003956564819967 """ I_check_minus_with_UInt128_max_and_min_value = r""" -minus(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) minus(toUInt128(\'0\'), toUInt128(1)) +round(minus(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(minus(toUInt128(\'0\'), toUInt128(1)), 7) -2 -1 """ I_check_minus_with_UInt256_max_and_min_value = r""" -minus(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) minus(toUInt256(\'0\'), toUInt256(1)) +round(minus(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(minus(toUInt256(\'0\'), toUInt256(1)), 7) -2 -1 """ I_check_multiply_with_Int128_max_and_min_value = r""" -multiply(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) multiply(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(multiply(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(multiply(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 170141183460469231731687303715884105727 -170141183460469231731687303715884105728 """ I_check_multiply_with_Int256_max_and_min_value = r""" -multiply(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) multiply(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(multiply(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(multiply(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 57896044618658097711785492504343953926634992332820282019728792003956564819967 -57896044618658097711785492504343953926634992332820282019728792003956564819968 """ I_check_multiply_with_UInt128_max_and_min_value = r""" -multiply(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) multiply(toUInt128(\'0\'), toUInt128(1)) +round(multiply(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(multiply(toUInt128(\'0\'), toUInt128(1)), 7) 340282366920938463463374607431768211455 0 """ I_check_multiply_with_UInt256_max_and_min_value = r""" -multiply(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) multiply(toUInt256(\'0\'), toUInt256(1)) +round(multiply(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(multiply(toUInt256(\'0\'), toUInt256(1)), 7) 115792089237316195423570985008687907853269984665640564039457584007913129639935 0 """ I_check_divide_with_Int128_max_and_min_value = r""" -divide(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) divide(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(divide(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(divide(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 1.7014118346046923e38 -1.7014118346046923e38 """ I_check_divide_with_Int256_max_and_min_value = r""" -divide(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) divide(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(divide(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(divide(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 5.78960446186581e76 -5.78960446186581e76 """ I_check_divide_with_UInt128_max_and_min_value = r""" -divide(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) divide(toUInt128(\'0\'), toUInt128(1)) +round(divide(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(divide(toUInt128(\'0\'), toUInt128(1)), 7) 3.402823669209385e38 0 """ I_check_divide_with_UInt256_max_and_min_value = r""" -divide(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) divide(toUInt256(\'0\'), toUInt256(1)) +round(divide(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(divide(toUInt256(\'0\'), toUInt256(1)), 7) 1.157920892373162e77 0 """ I_check_intDiv_with_Int128_max_and_min_value = r""" -intDiv(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) intDiv(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(intDiv(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(intDiv(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 170141183460469231731687303715884105727 -170141183460469231731687303715884105728 """ I_check_intDiv_with_Int256_max_and_min_value = r""" -intDiv(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) intDiv(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(intDiv(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(intDiv(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 57896044618658097711785492504343953926634992332820282019728792003956564819967 -57896044618658097711785492504343953926634992332820282019728792003956564819968 """ I_check_intDiv_with_UInt128_max_and_min_value = r""" -intDiv(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) intDiv(toUInt128(\'0\'), toUInt128(1)) +round(intDiv(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(intDiv(toUInt128(\'0\'), toUInt128(1)), 7) 340282366920938463463374607431768211455 0 """ I_check_intDiv_with_UInt256_max_and_min_value = r""" -intDiv(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) intDiv(toUInt256(\'0\'), toUInt256(1)) +round(intDiv(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(intDiv(toUInt256(\'0\'), toUInt256(1)), 7) 115792089237316195423570985008687907853269984665640564039457584007913129639935 0 """ I_check_intDivOrZero_with_Int128_max_and_min_value = r""" -intDivOrZero(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) intDivOrZero(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(intDivOrZero(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(intDivOrZero(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 170141183460469231731687303715884105727 -170141183460469231731687303715884105728 """ I_check_intDivOrZero_with_Int256_max_and_min_value = r""" -intDivOrZero(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) intDivOrZero(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(intDivOrZero(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(intDivOrZero(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 57896044618658097711785492504343953926634992332820282019728792003956564819967 -57896044618658097711785492504343953926634992332820282019728792003956564819968 """ I_check_intDivOrZero_with_UInt128_max_and_min_value = r""" -intDivOrZero(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) intDivOrZero(toUInt128(\'0\'), toUInt128(1)) +round(intDivOrZero(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(intDivOrZero(toUInt128(\'0\'), toUInt128(1)), 7) 340282366920938463463374607431768211455 0 """ I_check_intDivOrZero_with_UInt256_max_and_min_value = r""" -intDivOrZero(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) intDivOrZero(toUInt256(\'0\'), toUInt256(1)) +round(intDivOrZero(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(intDivOrZero(toUInt256(\'0\'), toUInt256(1)), 7) 115792089237316195423570985008687907853269984665640564039457584007913129639935 0 """ I_check_modulo_with_Int128_max_and_min_value = r""" -modulo(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) modulo(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(modulo(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(modulo(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 0 0 """ I_check_modulo_with_Int256_max_and_min_value = r""" -modulo(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) modulo(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(modulo(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(modulo(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 0 0 """ I_check_modulo_with_UInt128_max_and_min_value = r""" -modulo(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) modulo(toUInt128(\'0\'), toUInt128(1)) +round(modulo(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(modulo(toUInt128(\'0\'), toUInt128(1)), 7) 0 0 """ I_check_modulo_with_UInt256_max_and_min_value = r""" -modulo(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) modulo(toUInt256(\'0\'), toUInt256(1)) +round(modulo(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(modulo(toUInt256(\'0\'), toUInt256(1)), 7) 0 0 """ I_check_moduloOrZero_with_Int128_max_and_min_value = r""" -moduloOrZero(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)) moduloOrZero(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)) +round(moduloOrZero(toInt128(\'170141183460469231731687303715884105727\'), toInt128(1)), 7) round(moduloOrZero(toInt128(\'-170141183460469231731687303715884105728\'), toInt128(1)), 7) 0 0 """ I_check_moduloOrZero_with_Int256_max_and_min_value = r""" -moduloOrZero(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)) moduloOrZero(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)) +round(moduloOrZero(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\'), toInt256(1)), 7) round(moduloOrZero(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\'), toInt256(1)), 7) 0 0 """ I_check_moduloOrZero_with_UInt128_max_and_min_value = r""" -moduloOrZero(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)) moduloOrZero(toUInt128(\'0\'), toUInt128(1)) +round(moduloOrZero(toUInt128(\'340282366920938463463374607431768211455\'), toUInt128(1)), 7) round(moduloOrZero(toUInt128(\'0\'), toUInt128(1)), 7) 0 0 """ I_check_moduloOrZero_with_UInt256_max_and_min_value = r""" -moduloOrZero(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)) moduloOrZero(toUInt256(\'0\'), toUInt256(1)) +round(moduloOrZero(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\'), toUInt256(1)), 7) round(moduloOrZero(toUInt256(\'0\'), toUInt256(1)), 7) 0 0 """ @@ -532,399 +532,399 @@ a 1 """ -Inline___Int128___arrayPopBack_ = r""" +Inline___Int128___arrayPopBack__ = r""" arrayPopBack(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2] """ -Table___Int128___arrayPopBack_ = r""" +Table___Int128___arrayPopBack__ = r""" a [3,2] """ -Inline___Int128___arrayPopFront_ = r""" +Inline___Int128___arrayPopFront__ = r""" arrayPopFront(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [2,1] """ -Table___Int128___arrayPopFront_ = r""" +Table___Int128___arrayPopFront__ = r""" a [2,1] """ -Inline___Int128___arraySort_ = r""" +Inline___Int128___arraySort__ = r""" arraySort(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,2,3] """ -Table___Int128___arraySort_ = r""" +Table___Int128___arraySort__ = r""" a [1,2,3] """ -Inline___Int128___arrayReverseSort_ = r""" +Inline___Int128___arrayReverseSort__ = r""" arrayReverseSort(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2,1] """ -Table___Int128___arrayReverseSort_ = r""" +Table___Int128___arrayReverseSort__ = r""" a [3,2,1] """ -Inline___Int128___arrayDistinct_ = r""" +Inline___Int128___arrayDistinct__ = r""" arrayDistinct(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2,1] """ -Table___Int128___arrayDistinct_ = r""" +Table___Int128___arrayDistinct__ = r""" a [3,2,1] """ -Inline___Int128___arrayEnumerate_ = r""" +Inline___Int128___arrayEnumerate__ = r""" arrayEnumerate(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,2,3] """ -Table___Int128___arrayEnumerate_ = r""" +Table___Int128___arrayEnumerate__ = r""" a [1,2,3] """ -Inline___Int128___arrayEnumerateDense_ = r""" +Inline___Int128___arrayEnumerateDense__ = r""" arrayEnumerateDense(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,2,3] """ -Table___Int128___arrayEnumerateDense_ = r""" +Table___Int128___arrayEnumerateDense__ = r""" a [1,2,3] """ -Inline___Int128___arrayEnumerateUniq_ = r""" +Inline___Int128___arrayEnumerateUniq__ = r""" arrayEnumerateUniq(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,1,1] """ -Table___Int128___arrayEnumerateUniq_ = r""" +Table___Int128___arrayEnumerateUniq__ = r""" a [1,1,1] """ -Inline___Int128___arrayReverse_ = r""" +Inline___Int128___arrayReverse__ = r""" arrayReverse(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,2,3] """ -Table___Int128___arrayReverse_ = r""" +Table___Int128___arrayReverse__ = r""" a [1,2,3] """ -Inline___Int128___reverse_ = r""" +Inline___Int128___reverse__ = r""" reverse(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1,2,3] """ -Table___Int128___reverse_ = r""" +Table___Int128___reverse__ = r""" a [1,2,3] """ -Inline___Int128___arrayFlatten_ = r""" +Inline___Int128___arrayFlatten__ = r""" arrayFlatten(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2,1] """ -Table___Int128___arrayFlatten_ = r""" +Table___Int128___arrayFlatten__ = r""" a [3,2,1] """ -Inline___Int128___arrayCompact_ = r""" +Inline___Int128___arrayCompact__ = r""" arrayCompact(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2,1] """ -Table___Int128___arrayCompact_ = r""" +Table___Int128___arrayCompact__ = r""" a [3,2,1] """ -Inline___Int128___arrayReduceInRanges__sum_____1__5___ = r""" +Inline___Int128___arrayReduceInRanges__sum_____1__5____ = r""" arrayReduceInRanges(\'sum\', array(tuple(1, 5)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [6] """ -Table___Int128___arrayReduceInRanges__sum_____1__5___ = r""" +Table___Int128___arrayReduceInRanges__sum_____1__5____ = r""" a [6] """ -Inline___Int128___arrayMap_x_____x___2__ = r""" +Inline___Int128___arrayMap_x_____x___2___ = r""" arrayMap(lambda(tuple(x), plus(x, 2)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [5,4,3] """ -Table___Int128___arrayMap_x_____x___2__ = r""" +Table___Int128___arrayMap_x_____x___2___ = r""" a [5,4,3] """ -Inline___Int128___arrayFill_x____x_3_ = r""" +Inline___Int128___arrayFill_x____x_3__ = r""" arrayFill(lambda(tuple(x), equals(x, 3)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,3,3] """ -Table___Int128___arrayFill_x____x_3_ = r""" +Table___Int128___arrayFill_x____x_3__ = r""" a [3,3,3] """ -Inline___Int128___arrayReverseFill_x____x_3_ = r""" +Inline___Int128___arrayReverseFill_x____x_3__ = r""" arrayReverseFill(lambda(tuple(x), equals(x, 3)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,1,1] """ -Table___Int128___arrayReverseFill_x____x_3_ = r""" +Table___Int128___arrayReverseFill_x____x_3__ = r""" a [3,1,1] """ -Inline___Int128___arrayConcat__toInt128__3____toInt128__2____toInt128__1____ = r""" +Inline___Int128___arrayConcat__toInt128__3____toInt128__2____toInt128__1_____ = r""" arrayConcat(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\')), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [3,2,1,3,2,1] """ -Table___Int128___arrayConcat__toInt128__3____toInt128__2____toInt128__1____ = r""" +Table___Int128___arrayConcat__toInt128__3____toInt128__2____toInt128__1_____ = r""" a [3,2,1,3,2,1] """ -Inline___Int128___arrayFilter_x____x____1__ = r""" +Inline___Int128___arrayFilter_x____x____1___ = r""" arrayFilter(lambda(tuple(x), equals(x, 1)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [1] """ -Table___Int128___arrayFilter_x____x____1__ = r""" +Table___Int128___arrayFilter_x____x____1___ = r""" a [1] """ -Inline___Int128___arraySplit__x__y_____x_y___0__0__0__ = r""" +Inline___Int128___arraySplit__x__y_____x_y___0__0__0___ = r""" arraySplit(lambda(tuple(x, y), equals(x, y)), [0, 0, 0], array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) [[0,0,0]] """ -Table___Int128___arraySplit__x__y_____x_y___0__0__0__ = r""" +Table___Int128___arraySplit__x__y_____x_y___0__0__0___ = r""" a [[0,0,0]] """ -Inline___Int128___arrayZip__toInt128__1____ = r""" +Inline___Int128___arrayZip__toInt128__1_____ = r""" arrayZip(array(toInt128(\'1\')), array(toInt128(\'3\'))) [(1,3)] """ -Table___Int128___arrayZip__toInt128__1____ = r""" +Table___Int128___arrayZip__toInt128__1_____ = r""" a [(1,1)] """ -Inline___Int128___empty_ = r""" +Inline___Int128___empty__ = r""" empty(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 0 """ -Table___Int128___empty_ = r""" +Table___Int128___empty__ = r""" a 0 """ -Inline___Int128___notEmpty_ = r""" +Inline___Int128___notEmpty__ = r""" notEmpty(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___notEmpty_ = r""" +Table___Int128___notEmpty__ = r""" a 1 """ -Inline___Int128___length_ = r""" +Inline___Int128___length__ = r""" length(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 """ -Table___Int128___length_ = r""" +Table___Int128___length__ = r""" a 3 """ -Inline___Int128___arrayCount_x____x____1__ = r""" +Inline___Int128___arrayCount_x____x____1___ = r""" arrayCount(lambda(tuple(x), equals(x, 1)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___arrayCount_x____x____1__ = r""" +Table___Int128___arrayCount_x____x____1___ = r""" a 1 """ -Inline___Int128___arrayUniq_ = r""" +Inline___Int128___arrayUniq__ = r""" arrayUniq(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 """ -Table___Int128___arrayUniq_ = r""" +Table___Int128___arrayUniq__ = r""" a 3 """ -Inline___Int128___arrayJoin_ = r""" +Inline___Int128___arrayJoin__ = r""" arrayJoin(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 2 1 """ -Table___Int128___arrayJoin_ = r""" +Table___Int128___arrayJoin__ = r""" a 1 2 3 """ -Inline___Int128___arrayExists_x____x__1_ = r""" +Inline___Int128___arrayExists_x____x__1__ = r""" arrayExists(lambda(tuple(x), equals(x, 1)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___arrayExists_x____x__1_ = r""" +Table___Int128___arrayExists_x____x__1__ = r""" a 1 """ -Inline___Int128___arrayAll_x____x__1_ = r""" +Inline___Int128___arrayAll_x____x__1__ = r""" arrayAll(lambda(tuple(x), equals(x, 1)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 0 """ -Table___Int128___arrayAll_x____x__1_ = r""" +Table___Int128___arrayAll_x____x__1__ = r""" a 0 """ -Inline___Int128___arrayMin_ = r""" +Inline___Int128___arrayMin__ = r""" arrayMin(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___arrayMin_ = r""" +Table___Int128___arrayMin__ = r""" a 1 """ -Inline___Int128___arrayMax_ = r""" +Inline___Int128___arrayMax__ = r""" arrayMax(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 """ -Table___Int128___arrayMax_ = r""" +Table___Int128___arrayMax__ = r""" a 3 """ -Inline___Int128___arraySum_ = r""" +Inline___Int128___arraySum__ = r""" arraySum(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 6 """ -Table___Int128___arraySum_ = r""" +Table___Int128___arraySum__ = r""" a 6 """ -Inline___Int128___arrayAvg_ = r""" +Inline___Int128___arrayAvg__ = r""" arrayAvg(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 2 """ -Table___Int128___arrayAvg_ = r""" +Table___Int128___arrayAvg__ = r""" a 2 """ -Inline___Int128___arrayReduce__max___ = r""" +Inline___Int128___arrayReduce__max____ = r""" arrayReduce(\'max\', array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 """ -Table___Int128___arrayReduce__max___ = r""" +Table___Int128___arrayReduce__max____ = r""" a 3 """ -Inline___Int128___arrayFirst_x____x__3_ = r""" +Inline___Int128___arrayFirst_x____x__3__ = r""" arrayFirst(lambda(tuple(x), equals(x, 3)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 3 """ -Table___Int128___arrayFirst_x____x__3_ = r""" +Table___Int128___arrayFirst_x____x__3__ = r""" a 3 """ -Inline___Int128___arrayFirstIndex_x____x__3_ = r""" +Inline___Int128___arrayFirstIndex_x____x__3__ = r""" arrayFirstIndex(lambda(tuple(x), equals(x, 3)), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___arrayFirstIndex_x____x__3_ = r""" +Table___Int128___arrayFirstIndex_x____x__3__ = r""" a 1 """ -Inline___Int128___hasAll__toInt128__3____toInt128__2____toInt128__1_____ = r""" +Inline___Int128___hasAll__toInt128__3____toInt128__2____toInt128__1______ = r""" hasAll(array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\')), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___hasAll__toInt128__3____toInt128__2____toInt128__1_____ = r""" +Table___Int128___hasAll__toInt128__3____toInt128__2____toInt128__1______ = r""" a 1 """ -Inline___Int128___hasAny__toInt128__2____toInt128__1_____ = r""" +Inline___Int128___hasAny__toInt128__2____toInt128__1______ = r""" hasAny(array(toInt128(\'2\'), toInt128(\'1\')), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 1 """ -Table___Int128___hasAny__toInt128__2____toInt128__1_____ = r""" +Table___Int128___hasAny__toInt128__2____toInt128__1______ = r""" a 1 """ -Inline___Int128___hasSubstr__toInt128__2____toInt128__1_____ = r""" +Inline___Int128___hasSubstr__toInt128__2____toInt128__1______ = r""" hasSubstr(array(toInt128(\'2\'), toInt128(\'1\')), array(toInt128(\'3\'), toInt128(\'2\'), toInt128(\'1\'))) 0 """ -Table___Int128___hasSubstr__toInt128__2____toInt128__1_____ = r""" +Table___Int128___hasSubstr__toInt128__2____toInt128__1______ = r""" a 0 """ -Table___Int128___arrayDifference_ = r""" +Table___Int128___arrayDifference__ = r""" a """ -Table___Int128___arrayCumSum_ = r""" +Table___Int128___arrayCumSum__ = r""" a """ -Table___Int128___arrayCumSumNonNegative_ = r""" +Table___Int128___arrayCumSumNonNegative__ = r""" a """ @@ -1060,399 +1060,399 @@ a [1,2] """ -Inline___Int256___arrayPopBack_ = r""" +Inline___Int256___arrayPopBack__ = r""" arrayPopBack(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2] """ -Table___Int256___arrayPopBack_ = r""" +Table___Int256___arrayPopBack__ = r""" a [3,2] """ -Inline___Int256___arrayPopFront_ = r""" +Inline___Int256___arrayPopFront__ = r""" arrayPopFront(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [2,1] """ -Table___Int256___arrayPopFront_ = r""" +Table___Int256___arrayPopFront__ = r""" a [2,1] """ -Inline___Int256___arraySort_ = r""" +Inline___Int256___arraySort__ = r""" arraySort(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,2,3] """ -Table___Int256___arraySort_ = r""" +Table___Int256___arraySort__ = r""" a [1,2,3] """ -Inline___Int256___arrayReverseSort_ = r""" +Inline___Int256___arrayReverseSort__ = r""" arrayReverseSort(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2,1] """ -Table___Int256___arrayReverseSort_ = r""" +Table___Int256___arrayReverseSort__ = r""" a [3,2,1] """ -Inline___Int256___arrayDistinct_ = r""" +Inline___Int256___arrayDistinct__ = r""" arrayDistinct(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2,1] """ -Table___Int256___arrayDistinct_ = r""" +Table___Int256___arrayDistinct__ = r""" a [3,2,1] """ -Inline___Int256___arrayEnumerate_ = r""" +Inline___Int256___arrayEnumerate__ = r""" arrayEnumerate(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,2,3] """ -Table___Int256___arrayEnumerate_ = r""" +Table___Int256___arrayEnumerate__ = r""" a [1,2,3] """ -Inline___Int256___arrayEnumerateDense_ = r""" +Inline___Int256___arrayEnumerateDense__ = r""" arrayEnumerateDense(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,2,3] """ -Table___Int256___arrayEnumerateDense_ = r""" +Table___Int256___arrayEnumerateDense__ = r""" a [1,2,3] """ -Inline___Int256___arrayEnumerateUniq_ = r""" +Inline___Int256___arrayEnumerateUniq__ = r""" arrayEnumerateUniq(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,1,1] """ -Table___Int256___arrayEnumerateUniq_ = r""" +Table___Int256___arrayEnumerateUniq__ = r""" a [1,1,1] """ -Inline___Int256___arrayReverse_ = r""" +Inline___Int256___arrayReverse__ = r""" arrayReverse(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,2,3] """ -Table___Int256___arrayReverse_ = r""" +Table___Int256___arrayReverse__ = r""" a [1,2,3] """ -Inline___Int256___reverse_ = r""" +Inline___Int256___reverse__ = r""" reverse(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1,2,3] """ -Table___Int256___reverse_ = r""" +Table___Int256___reverse__ = r""" a [1,2,3] """ -Inline___Int256___arrayFlatten_ = r""" +Inline___Int256___arrayFlatten__ = r""" arrayFlatten(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2,1] """ -Table___Int256___arrayFlatten_ = r""" +Table___Int256___arrayFlatten__ = r""" a [3,2,1] """ -Inline___Int256___arrayCompact_ = r""" +Inline___Int256___arrayCompact__ = r""" arrayCompact(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2,1] """ -Table___Int256___arrayCompact_ = r""" +Table___Int256___arrayCompact__ = r""" a [3,2,1] """ -Inline___Int256___arrayReduceInRanges__sum_____1__5___ = r""" +Inline___Int256___arrayReduceInRanges__sum_____1__5____ = r""" arrayReduceInRanges(\'sum\', array(tuple(1, 5)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [6] """ -Table___Int256___arrayReduceInRanges__sum_____1__5___ = r""" +Table___Int256___arrayReduceInRanges__sum_____1__5____ = r""" a [6] """ -Inline___Int256___arrayMap_x_____x___2__ = r""" +Inline___Int256___arrayMap_x_____x___2___ = r""" arrayMap(lambda(tuple(x), plus(x, 2)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [5,4,3] """ -Table___Int256___arrayMap_x_____x___2__ = r""" +Table___Int256___arrayMap_x_____x___2___ = r""" a [5,4,3] """ -Inline___Int256___arrayFill_x____x_3_ = r""" +Inline___Int256___arrayFill_x____x_3__ = r""" arrayFill(lambda(tuple(x), equals(x, 3)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,3,3] """ -Table___Int256___arrayFill_x____x_3_ = r""" +Table___Int256___arrayFill_x____x_3__ = r""" a [3,3,3] """ -Inline___Int256___arrayReverseFill_x____x_3_ = r""" +Inline___Int256___arrayReverseFill_x____x_3__ = r""" arrayReverseFill(lambda(tuple(x), equals(x, 3)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,1,1] """ -Table___Int256___arrayReverseFill_x____x_3_ = r""" +Table___Int256___arrayReverseFill_x____x_3__ = r""" a [3,1,1] """ -Inline___Int256___arrayConcat__toInt256__3____toInt256__2____toInt256__1____ = r""" +Inline___Int256___arrayConcat__toInt256__3____toInt256__2____toInt256__1_____ = r""" arrayConcat(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\')), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [3,2,1,3,2,1] """ -Table___Int256___arrayConcat__toInt256__3____toInt256__2____toInt256__1____ = r""" +Table___Int256___arrayConcat__toInt256__3____toInt256__2____toInt256__1_____ = r""" a [3,2,1,3,2,1] """ -Inline___Int256___arrayFilter_x____x____1__ = r""" +Inline___Int256___arrayFilter_x____x____1___ = r""" arrayFilter(lambda(tuple(x), equals(x, 1)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [1] """ -Table___Int256___arrayFilter_x____x____1__ = r""" +Table___Int256___arrayFilter_x____x____1___ = r""" a [1] """ -Inline___Int256___arraySplit__x__y_____x_y___0__0__0__ = r""" +Inline___Int256___arraySplit__x__y_____x_y___0__0__0___ = r""" arraySplit(lambda(tuple(x, y), equals(x, y)), [0, 0, 0], array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) [[0,0,0]] """ -Table___Int256___arraySplit__x__y_____x_y___0__0__0__ = r""" +Table___Int256___arraySplit__x__y_____x_y___0__0__0___ = r""" a [[0,0,0]] """ -Inline___Int256___arrayZip__toInt256__1____ = r""" +Inline___Int256___arrayZip__toInt256__1_____ = r""" arrayZip(array(toInt256(\'1\')), array(toInt256(\'3\'))) [(1,3)] """ -Table___Int256___arrayZip__toInt256__1____ = r""" +Table___Int256___arrayZip__toInt256__1_____ = r""" a [(1,1)] """ -Inline___Int256___empty_ = r""" +Inline___Int256___empty__ = r""" empty(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 0 """ -Table___Int256___empty_ = r""" +Table___Int256___empty__ = r""" a 0 """ -Inline___Int256___notEmpty_ = r""" +Inline___Int256___notEmpty__ = r""" notEmpty(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___notEmpty_ = r""" +Table___Int256___notEmpty__ = r""" a 1 """ -Inline___Int256___length_ = r""" +Inline___Int256___length__ = r""" length(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 """ -Table___Int256___length_ = r""" +Table___Int256___length__ = r""" a 3 """ -Inline___Int256___arrayCount_x____x____1__ = r""" +Inline___Int256___arrayCount_x____x____1___ = r""" arrayCount(lambda(tuple(x), equals(x, 1)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___arrayCount_x____x____1__ = r""" +Table___Int256___arrayCount_x____x____1___ = r""" a 1 """ -Inline___Int256___arrayUniq_ = r""" +Inline___Int256___arrayUniq__ = r""" arrayUniq(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 """ -Table___Int256___arrayUniq_ = r""" +Table___Int256___arrayUniq__ = r""" a 3 """ -Inline___Int256___arrayJoin_ = r""" +Inline___Int256___arrayJoin__ = r""" arrayJoin(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 2 1 """ -Table___Int256___arrayJoin_ = r""" +Table___Int256___arrayJoin__ = r""" a 1 2 3 """ -Inline___Int256___arrayExists_x____x__1_ = r""" +Inline___Int256___arrayExists_x____x__1__ = r""" arrayExists(lambda(tuple(x), equals(x, 1)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___arrayExists_x____x__1_ = r""" +Table___Int256___arrayExists_x____x__1__ = r""" a 1 """ -Inline___Int256___arrayAll_x____x__1_ = r""" +Inline___Int256___arrayAll_x____x__1__ = r""" arrayAll(lambda(tuple(x), equals(x, 1)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 0 """ -Table___Int256___arrayAll_x____x__1_ = r""" +Table___Int256___arrayAll_x____x__1__ = r""" a 0 """ -Inline___Int256___arrayMin_ = r""" +Inline___Int256___arrayMin__ = r""" arrayMin(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___arrayMin_ = r""" +Table___Int256___arrayMin__ = r""" a 1 """ -Inline___Int256___arrayMax_ = r""" +Inline___Int256___arrayMax__ = r""" arrayMax(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 """ -Table___Int256___arrayMax_ = r""" +Table___Int256___arrayMax__ = r""" a 3 """ -Inline___Int256___arraySum_ = r""" +Inline___Int256___arraySum__ = r""" arraySum(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 6 """ -Table___Int256___arraySum_ = r""" +Table___Int256___arraySum__ = r""" a 6 """ -Inline___Int256___arrayAvg_ = r""" +Inline___Int256___arrayAvg__ = r""" arrayAvg(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 2 """ -Table___Int256___arrayAvg_ = r""" +Table___Int256___arrayAvg__ = r""" a 2 """ -Inline___Int256___arrayReduce__max___ = r""" +Inline___Int256___arrayReduce__max____ = r""" arrayReduce(\'max\', array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 """ -Table___Int256___arrayReduce__max___ = r""" +Table___Int256___arrayReduce__max____ = r""" a 3 """ -Inline___Int256___arrayFirst_x____x__3_ = r""" +Inline___Int256___arrayFirst_x____x__3__ = r""" arrayFirst(lambda(tuple(x), equals(x, 3)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 3 """ -Table___Int256___arrayFirst_x____x__3_ = r""" +Table___Int256___arrayFirst_x____x__3__ = r""" a 3 """ -Inline___Int256___arrayFirstIndex_x____x__3_ = r""" +Inline___Int256___arrayFirstIndex_x____x__3__ = r""" arrayFirstIndex(lambda(tuple(x), equals(x, 3)), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___arrayFirstIndex_x____x__3_ = r""" +Table___Int256___arrayFirstIndex_x____x__3__ = r""" a 1 """ -Inline___Int256___hasAll__toInt256__3____toInt256__2____toInt256__1_____ = r""" +Inline___Int256___hasAll__toInt256__3____toInt256__2____toInt256__1______ = r""" hasAll(array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\')), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___hasAll__toInt256__3____toInt256__2____toInt256__1_____ = r""" +Table___Int256___hasAll__toInt256__3____toInt256__2____toInt256__1______ = r""" a 1 """ -Inline___Int256___hasAny__toInt256__2____toInt256__1_____ = r""" +Inline___Int256___hasAny__toInt256__2____toInt256__1______ = r""" hasAny(array(toInt256(\'2\'), toInt256(\'1\')), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 1 """ -Table___Int256___hasAny__toInt256__2____toInt256__1_____ = r""" +Table___Int256___hasAny__toInt256__2____toInt256__1______ = r""" a 1 """ -Inline___Int256___hasSubstr__toInt256__2____toInt256__1_____ = r""" +Inline___Int256___hasSubstr__toInt256__2____toInt256__1______ = r""" hasSubstr(array(toInt256(\'2\'), toInt256(\'1\')), array(toInt256(\'3\'), toInt256(\'2\'), toInt256(\'1\'))) 0 """ -Table___Int256___hasSubstr__toInt256__2____toInt256__1_____ = r""" +Table___Int256___hasSubstr__toInt256__2____toInt256__1______ = r""" a 0 """ -Table___Int256___arrayDifference_ = r""" +Table___Int256___arrayDifference__ = r""" a """ -Table___Int256___arrayCumSum_ = r""" +Table___Int256___arrayCumSum__ = r""" a """ -Table___Int256___arrayCumSumNonNegative_ = r""" +Table___Int256___arrayCumSumNonNegative__ = r""" a """ @@ -1588,399 +1588,399 @@ a [1,2] """ -Inline___UInt128___arrayPopBack_ = r""" +Inline___UInt128___arrayPopBack__ = r""" arrayPopBack(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2] """ -Table___UInt128___arrayPopBack_ = r""" +Table___UInt128___arrayPopBack__ = r""" a [3,2] """ -Inline___UInt128___arrayPopFront_ = r""" +Inline___UInt128___arrayPopFront__ = r""" arrayPopFront(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [2,1] """ -Table___UInt128___arrayPopFront_ = r""" +Table___UInt128___arrayPopFront__ = r""" a [2,1] """ -Inline___UInt128___arraySort_ = r""" +Inline___UInt128___arraySort__ = r""" arraySort(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,2,3] """ -Table___UInt128___arraySort_ = r""" +Table___UInt128___arraySort__ = r""" a [1,2,3] """ -Inline___UInt128___arrayReverseSort_ = r""" +Inline___UInt128___arrayReverseSort__ = r""" arrayReverseSort(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2,1] """ -Table___UInt128___arrayReverseSort_ = r""" +Table___UInt128___arrayReverseSort__ = r""" a [3,2,1] """ -Inline___UInt128___arrayDistinct_ = r""" +Inline___UInt128___arrayDistinct__ = r""" arrayDistinct(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2,1] """ -Table___UInt128___arrayDistinct_ = r""" +Table___UInt128___arrayDistinct__ = r""" a [3,2,1] """ -Inline___UInt128___arrayEnumerate_ = r""" +Inline___UInt128___arrayEnumerate__ = r""" arrayEnumerate(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,2,3] """ -Table___UInt128___arrayEnumerate_ = r""" +Table___UInt128___arrayEnumerate__ = r""" a [1,2,3] """ -Inline___UInt128___arrayEnumerateDense_ = r""" +Inline___UInt128___arrayEnumerateDense__ = r""" arrayEnumerateDense(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,2,3] """ -Table___UInt128___arrayEnumerateDense_ = r""" +Table___UInt128___arrayEnumerateDense__ = r""" a [1,2,3] """ -Inline___UInt128___arrayEnumerateUniq_ = r""" +Inline___UInt128___arrayEnumerateUniq__ = r""" arrayEnumerateUniq(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,1,1] """ -Table___UInt128___arrayEnumerateUniq_ = r""" +Table___UInt128___arrayEnumerateUniq__ = r""" a [1,1,1] """ -Inline___UInt128___arrayReverse_ = r""" +Inline___UInt128___arrayReverse__ = r""" arrayReverse(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,2,3] """ -Table___UInt128___arrayReverse_ = r""" +Table___UInt128___arrayReverse__ = r""" a [1,2,3] """ -Inline___UInt128___reverse_ = r""" +Inline___UInt128___reverse__ = r""" reverse(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1,2,3] """ -Table___UInt128___reverse_ = r""" +Table___UInt128___reverse__ = r""" a [1,2,3] """ -Inline___UInt128___arrayFlatten_ = r""" +Inline___UInt128___arrayFlatten__ = r""" arrayFlatten(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2,1] """ -Table___UInt128___arrayFlatten_ = r""" +Table___UInt128___arrayFlatten__ = r""" a [3,2,1] """ -Inline___UInt128___arrayCompact_ = r""" +Inline___UInt128___arrayCompact__ = r""" arrayCompact(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2,1] """ -Table___UInt128___arrayCompact_ = r""" +Table___UInt128___arrayCompact__ = r""" a [3,2,1] """ -Inline___UInt128___arrayReduceInRanges__sum_____1__5___ = r""" +Inline___UInt128___arrayReduceInRanges__sum_____1__5____ = r""" arrayReduceInRanges(\'sum\', array(tuple(1, 5)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [6] """ -Table___UInt128___arrayReduceInRanges__sum_____1__5___ = r""" +Table___UInt128___arrayReduceInRanges__sum_____1__5____ = r""" a [6] """ -Inline___UInt128___arrayMap_x_____x___2__ = r""" +Inline___UInt128___arrayMap_x_____x___2___ = r""" arrayMap(lambda(tuple(x), plus(x, 2)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [5,4,3] """ -Table___UInt128___arrayMap_x_____x___2__ = r""" +Table___UInt128___arrayMap_x_____x___2___ = r""" a [5,4,3] """ -Inline___UInt128___arrayFill_x____x_3_ = r""" +Inline___UInt128___arrayFill_x____x_3__ = r""" arrayFill(lambda(tuple(x), equals(x, 3)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,3,3] """ -Table___UInt128___arrayFill_x____x_3_ = r""" +Table___UInt128___arrayFill_x____x_3__ = r""" a [3,3,3] """ -Inline___UInt128___arrayReverseFill_x____x_3_ = r""" +Inline___UInt128___arrayReverseFill_x____x_3__ = r""" arrayReverseFill(lambda(tuple(x), equals(x, 3)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,1,1] """ -Table___UInt128___arrayReverseFill_x____x_3_ = r""" +Table___UInt128___arrayReverseFill_x____x_3__ = r""" a [3,1,1] """ -Inline___UInt128___arrayConcat__toUInt128__3____toUInt128__2____toUInt128__1____ = r""" +Inline___UInt128___arrayConcat__toUInt128__3____toUInt128__2____toUInt128__1_____ = r""" arrayConcat(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\')), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [3,2,1,3,2,1] """ -Table___UInt128___arrayConcat__toUInt128__3____toUInt128__2____toUInt128__1____ = r""" +Table___UInt128___arrayConcat__toUInt128__3____toUInt128__2____toUInt128__1_____ = r""" a [3,2,1,3,2,1] """ -Inline___UInt128___arrayFilter_x____x____1__ = r""" +Inline___UInt128___arrayFilter_x____x____1___ = r""" arrayFilter(lambda(tuple(x), equals(x, 1)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [1] """ -Table___UInt128___arrayFilter_x____x____1__ = r""" +Table___UInt128___arrayFilter_x____x____1___ = r""" a [1] """ -Inline___UInt128___arraySplit__x__y_____x_y___0__0__0__ = r""" +Inline___UInt128___arraySplit__x__y_____x_y___0__0__0___ = r""" arraySplit(lambda(tuple(x, y), equals(x, y)), [0, 0, 0], array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) [[0,0,0]] """ -Table___UInt128___arraySplit__x__y_____x_y___0__0__0__ = r""" +Table___UInt128___arraySplit__x__y_____x_y___0__0__0___ = r""" a [[0,0,0]] """ -Inline___UInt128___arrayZip__toUInt128__1____ = r""" +Inline___UInt128___arrayZip__toUInt128__1_____ = r""" arrayZip(array(toUInt128(\'1\')), array(toUInt128(\'3\'))) [(1,3)] """ -Table___UInt128___arrayZip__toUInt128__1____ = r""" +Table___UInt128___arrayZip__toUInt128__1_____ = r""" a [(1,1)] """ -Inline___UInt128___empty_ = r""" +Inline___UInt128___empty__ = r""" empty(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 0 """ -Table___UInt128___empty_ = r""" +Table___UInt128___empty__ = r""" a 0 """ -Inline___UInt128___notEmpty_ = r""" +Inline___UInt128___notEmpty__ = r""" notEmpty(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___notEmpty_ = r""" +Table___UInt128___notEmpty__ = r""" a 1 """ -Inline___UInt128___length_ = r""" +Inline___UInt128___length__ = r""" length(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 """ -Table___UInt128___length_ = r""" +Table___UInt128___length__ = r""" a 3 """ -Inline___UInt128___arrayCount_x____x____1__ = r""" +Inline___UInt128___arrayCount_x____x____1___ = r""" arrayCount(lambda(tuple(x), equals(x, 1)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___arrayCount_x____x____1__ = r""" +Table___UInt128___arrayCount_x____x____1___ = r""" a 1 """ -Inline___UInt128___arrayUniq_ = r""" +Inline___UInt128___arrayUniq__ = r""" arrayUniq(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 """ -Table___UInt128___arrayUniq_ = r""" +Table___UInt128___arrayUniq__ = r""" a 3 """ -Inline___UInt128___arrayJoin_ = r""" +Inline___UInt128___arrayJoin__ = r""" arrayJoin(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 2 1 """ -Table___UInt128___arrayJoin_ = r""" +Table___UInt128___arrayJoin__ = r""" a 1 2 3 """ -Inline___UInt128___arrayExists_x____x__1_ = r""" +Inline___UInt128___arrayExists_x____x__1__ = r""" arrayExists(lambda(tuple(x), equals(x, 1)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___arrayExists_x____x__1_ = r""" +Table___UInt128___arrayExists_x____x__1__ = r""" a 1 """ -Inline___UInt128___arrayAll_x____x__1_ = r""" +Inline___UInt128___arrayAll_x____x__1__ = r""" arrayAll(lambda(tuple(x), equals(x, 1)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 0 """ -Table___UInt128___arrayAll_x____x__1_ = r""" +Table___UInt128___arrayAll_x____x__1__ = r""" a 0 """ -Inline___UInt128___arrayMin_ = r""" +Inline___UInt128___arrayMin__ = r""" arrayMin(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___arrayMin_ = r""" +Table___UInt128___arrayMin__ = r""" a 1 """ -Inline___UInt128___arrayMax_ = r""" +Inline___UInt128___arrayMax__ = r""" arrayMax(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 """ -Table___UInt128___arrayMax_ = r""" +Table___UInt128___arrayMax__ = r""" a 3 """ -Inline___UInt128___arraySum_ = r""" +Inline___UInt128___arraySum__ = r""" arraySum(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 6 """ -Table___UInt128___arraySum_ = r""" +Table___UInt128___arraySum__ = r""" a 6 """ -Inline___UInt128___arrayAvg_ = r""" +Inline___UInt128___arrayAvg__ = r""" arrayAvg(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 2 """ -Table___UInt128___arrayAvg_ = r""" +Table___UInt128___arrayAvg__ = r""" a 2 """ -Inline___UInt128___arrayReduce__max___ = r""" +Inline___UInt128___arrayReduce__max____ = r""" arrayReduce(\'max\', array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 """ -Table___UInt128___arrayReduce__max___ = r""" +Table___UInt128___arrayReduce__max____ = r""" a 3 """ -Inline___UInt128___arrayFirst_x____x__3_ = r""" +Inline___UInt128___arrayFirst_x____x__3__ = r""" arrayFirst(lambda(tuple(x), equals(x, 3)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 3 """ -Table___UInt128___arrayFirst_x____x__3_ = r""" +Table___UInt128___arrayFirst_x____x__3__ = r""" a 3 """ -Inline___UInt128___arrayFirstIndex_x____x__3_ = r""" +Inline___UInt128___arrayFirstIndex_x____x__3__ = r""" arrayFirstIndex(lambda(tuple(x), equals(x, 3)), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___arrayFirstIndex_x____x__3_ = r""" +Table___UInt128___arrayFirstIndex_x____x__3__ = r""" a 1 """ -Inline___UInt128___hasAll__toUInt128__3____toUInt128__2____toUInt128__1_____ = r""" +Inline___UInt128___hasAll__toUInt128__3____toUInt128__2____toUInt128__1______ = r""" hasAll(array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\')), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___hasAll__toUInt128__3____toUInt128__2____toUInt128__1_____ = r""" +Table___UInt128___hasAll__toUInt128__3____toUInt128__2____toUInt128__1______ = r""" a 1 """ -Inline___UInt128___hasAny__toUInt128__2____toUInt128__1_____ = r""" +Inline___UInt128___hasAny__toUInt128__2____toUInt128__1______ = r""" hasAny(array(toUInt128(\'2\'), toUInt128(\'1\')), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 1 """ -Table___UInt128___hasAny__toUInt128__2____toUInt128__1_____ = r""" +Table___UInt128___hasAny__toUInt128__2____toUInt128__1______ = r""" a 1 """ -Inline___UInt128___hasSubstr__toUInt128__2____toUInt128__1_____ = r""" +Inline___UInt128___hasSubstr__toUInt128__2____toUInt128__1______ = r""" hasSubstr(array(toUInt128(\'2\'), toUInt128(\'1\')), array(toUInt128(\'3\'), toUInt128(\'2\'), toUInt128(\'1\'))) 0 """ -Table___UInt128___hasSubstr__toUInt128__2____toUInt128__1_____ = r""" +Table___UInt128___hasSubstr__toUInt128__2____toUInt128__1______ = r""" a 0 """ -Table___UInt128___arrayDifference_ = r""" +Table___UInt128___arrayDifference__ = r""" a """ -Table___UInt128___arrayCumSum_ = r""" +Table___UInt128___arrayCumSum__ = r""" a """ -Table___UInt128___arrayCumSumNonNegative_ = r""" +Table___UInt128___arrayCumSumNonNegative__ = r""" a """ @@ -2116,399 +2116,399 @@ a [1,2] """ -Inline___UInt256___arrayPopBack_ = r""" +Inline___UInt256___arrayPopBack__ = r""" arrayPopBack(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2] """ -Table___UInt256___arrayPopBack_ = r""" +Table___UInt256___arrayPopBack__ = r""" a [3,2] """ -Inline___UInt256___arrayPopFront_ = r""" +Inline___UInt256___arrayPopFront__ = r""" arrayPopFront(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [2,1] """ -Table___UInt256___arrayPopFront_ = r""" +Table___UInt256___arrayPopFront__ = r""" a [2,1] """ -Inline___UInt256___arraySort_ = r""" +Inline___UInt256___arraySort__ = r""" arraySort(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,2,3] """ -Table___UInt256___arraySort_ = r""" +Table___UInt256___arraySort__ = r""" a [1,2,3] """ -Inline___UInt256___arrayReverseSort_ = r""" +Inline___UInt256___arrayReverseSort__ = r""" arrayReverseSort(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2,1] """ -Table___UInt256___arrayReverseSort_ = r""" +Table___UInt256___arrayReverseSort__ = r""" a [3,2,1] """ -Inline___UInt256___arrayDistinct_ = r""" +Inline___UInt256___arrayDistinct__ = r""" arrayDistinct(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2,1] """ -Table___UInt256___arrayDistinct_ = r""" +Table___UInt256___arrayDistinct__ = r""" a [3,2,1] """ -Inline___UInt256___arrayEnumerate_ = r""" +Inline___UInt256___arrayEnumerate__ = r""" arrayEnumerate(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,2,3] """ -Table___UInt256___arrayEnumerate_ = r""" +Table___UInt256___arrayEnumerate__ = r""" a [1,2,3] """ -Inline___UInt256___arrayEnumerateDense_ = r""" +Inline___UInt256___arrayEnumerateDense__ = r""" arrayEnumerateDense(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,2,3] """ -Table___UInt256___arrayEnumerateDense_ = r""" +Table___UInt256___arrayEnumerateDense__ = r""" a [1,2,3] """ -Inline___UInt256___arrayEnumerateUniq_ = r""" +Inline___UInt256___arrayEnumerateUniq__ = r""" arrayEnumerateUniq(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,1,1] """ -Table___UInt256___arrayEnumerateUniq_ = r""" +Table___UInt256___arrayEnumerateUniq__ = r""" a [1,1,1] """ -Inline___UInt256___arrayReverse_ = r""" +Inline___UInt256___arrayReverse__ = r""" arrayReverse(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,2,3] """ -Table___UInt256___arrayReverse_ = r""" +Table___UInt256___arrayReverse__ = r""" a [1,2,3] """ -Inline___UInt256___reverse_ = r""" +Inline___UInt256___reverse__ = r""" reverse(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1,2,3] """ -Table___UInt256___reverse_ = r""" +Table___UInt256___reverse__ = r""" a [1,2,3] """ -Inline___UInt256___arrayFlatten_ = r""" +Inline___UInt256___arrayFlatten__ = r""" arrayFlatten(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2,1] """ -Table___UInt256___arrayFlatten_ = r""" +Table___UInt256___arrayFlatten__ = r""" a [3,2,1] """ -Inline___UInt256___arrayCompact_ = r""" +Inline___UInt256___arrayCompact__ = r""" arrayCompact(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2,1] """ -Table___UInt256___arrayCompact_ = r""" +Table___UInt256___arrayCompact__ = r""" a [3,2,1] """ -Inline___UInt256___arrayReduceInRanges__sum_____1__5___ = r""" +Inline___UInt256___arrayReduceInRanges__sum_____1__5____ = r""" arrayReduceInRanges(\'sum\', array(tuple(1, 5)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [6] """ -Table___UInt256___arrayReduceInRanges__sum_____1__5___ = r""" +Table___UInt256___arrayReduceInRanges__sum_____1__5____ = r""" a [6] """ -Inline___UInt256___arrayMap_x_____x___2__ = r""" +Inline___UInt256___arrayMap_x_____x___2___ = r""" arrayMap(lambda(tuple(x), plus(x, 2)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [5,4,3] """ -Table___UInt256___arrayMap_x_____x___2__ = r""" +Table___UInt256___arrayMap_x_____x___2___ = r""" a [5,4,3] """ -Inline___UInt256___arrayFill_x____x_3_ = r""" +Inline___UInt256___arrayFill_x____x_3__ = r""" arrayFill(lambda(tuple(x), equals(x, 3)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,3,3] """ -Table___UInt256___arrayFill_x____x_3_ = r""" +Table___UInt256___arrayFill_x____x_3__ = r""" a [3,3,3] """ -Inline___UInt256___arrayReverseFill_x____x_3_ = r""" +Inline___UInt256___arrayReverseFill_x____x_3__ = r""" arrayReverseFill(lambda(tuple(x), equals(x, 3)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,1,1] """ -Table___UInt256___arrayReverseFill_x____x_3_ = r""" +Table___UInt256___arrayReverseFill_x____x_3__ = r""" a [3,1,1] """ -Inline___UInt256___arrayConcat__toUInt256__3____toUInt256__2____toUInt256__1____ = r""" +Inline___UInt256___arrayConcat__toUInt256__3____toUInt256__2____toUInt256__1_____ = r""" arrayConcat(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\')), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [3,2,1,3,2,1] """ -Table___UInt256___arrayConcat__toUInt256__3____toUInt256__2____toUInt256__1____ = r""" +Table___UInt256___arrayConcat__toUInt256__3____toUInt256__2____toUInt256__1_____ = r""" a [3,2,1,3,2,1] """ -Inline___UInt256___arrayFilter_x____x____1__ = r""" +Inline___UInt256___arrayFilter_x____x____1___ = r""" arrayFilter(lambda(tuple(x), equals(x, 1)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [1] """ -Table___UInt256___arrayFilter_x____x____1__ = r""" +Table___UInt256___arrayFilter_x____x____1___ = r""" a [1] """ -Inline___UInt256___arraySplit__x__y_____x_y___0__0__0__ = r""" +Inline___UInt256___arraySplit__x__y_____x_y___0__0__0___ = r""" arraySplit(lambda(tuple(x, y), equals(x, y)), [0, 0, 0], array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) [[0,0,0]] """ -Table___UInt256___arraySplit__x__y_____x_y___0__0__0__ = r""" +Table___UInt256___arraySplit__x__y_____x_y___0__0__0___ = r""" a [[0,0,0]] """ -Inline___UInt256___arrayZip__toUInt256__1____ = r""" +Inline___UInt256___arrayZip__toUInt256__1_____ = r""" arrayZip(array(toUInt256(\'1\')), array(toUInt256(\'3\'))) [(1,3)] """ -Table___UInt256___arrayZip__toUInt256__1____ = r""" +Table___UInt256___arrayZip__toUInt256__1_____ = r""" a [(1,1)] """ -Inline___UInt256___empty_ = r""" +Inline___UInt256___empty__ = r""" empty(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 0 """ -Table___UInt256___empty_ = r""" +Table___UInt256___empty__ = r""" a 0 """ -Inline___UInt256___notEmpty_ = r""" +Inline___UInt256___notEmpty__ = r""" notEmpty(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___notEmpty_ = r""" +Table___UInt256___notEmpty__ = r""" a 1 """ -Inline___UInt256___length_ = r""" +Inline___UInt256___length__ = r""" length(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 """ -Table___UInt256___length_ = r""" +Table___UInt256___length__ = r""" a 3 """ -Inline___UInt256___arrayCount_x____x____1__ = r""" +Inline___UInt256___arrayCount_x____x____1___ = r""" arrayCount(lambda(tuple(x), equals(x, 1)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___arrayCount_x____x____1__ = r""" +Table___UInt256___arrayCount_x____x____1___ = r""" a 1 """ -Inline___UInt256___arrayUniq_ = r""" +Inline___UInt256___arrayUniq__ = r""" arrayUniq(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 """ -Table___UInt256___arrayUniq_ = r""" +Table___UInt256___arrayUniq__ = r""" a 3 """ -Inline___UInt256___arrayJoin_ = r""" +Inline___UInt256___arrayJoin__ = r""" arrayJoin(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 2 1 """ -Table___UInt256___arrayJoin_ = r""" +Table___UInt256___arrayJoin__ = r""" a 1 2 3 """ -Inline___UInt256___arrayExists_x____x__1_ = r""" +Inline___UInt256___arrayExists_x____x__1__ = r""" arrayExists(lambda(tuple(x), equals(x, 1)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___arrayExists_x____x__1_ = r""" +Table___UInt256___arrayExists_x____x__1__ = r""" a 1 """ -Inline___UInt256___arrayAll_x____x__1_ = r""" +Inline___UInt256___arrayAll_x____x__1__ = r""" arrayAll(lambda(tuple(x), equals(x, 1)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 0 """ -Table___UInt256___arrayAll_x____x__1_ = r""" +Table___UInt256___arrayAll_x____x__1__ = r""" a 0 """ -Inline___UInt256___arrayMin_ = r""" +Inline___UInt256___arrayMin__ = r""" arrayMin(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___arrayMin_ = r""" +Table___UInt256___arrayMin__ = r""" a 1 """ -Inline___UInt256___arrayMax_ = r""" +Inline___UInt256___arrayMax__ = r""" arrayMax(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 """ -Table___UInt256___arrayMax_ = r""" +Table___UInt256___arrayMax__ = r""" a 3 """ -Inline___UInt256___arraySum_ = r""" +Inline___UInt256___arraySum__ = r""" arraySum(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 6 """ -Table___UInt256___arraySum_ = r""" +Table___UInt256___arraySum__ = r""" a 6 """ -Inline___UInt256___arrayAvg_ = r""" +Inline___UInt256___arrayAvg__ = r""" arrayAvg(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 2 """ -Table___UInt256___arrayAvg_ = r""" +Table___UInt256___arrayAvg__ = r""" a 2 """ -Inline___UInt256___arrayReduce__max___ = r""" +Inline___UInt256___arrayReduce__max____ = r""" arrayReduce(\'max\', array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 """ -Table___UInt256___arrayReduce__max___ = r""" +Table___UInt256___arrayReduce__max____ = r""" a 3 """ -Inline___UInt256___arrayFirst_x____x__3_ = r""" +Inline___UInt256___arrayFirst_x____x__3__ = r""" arrayFirst(lambda(tuple(x), equals(x, 3)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 3 """ -Table___UInt256___arrayFirst_x____x__3_ = r""" +Table___UInt256___arrayFirst_x____x__3__ = r""" a 3 """ -Inline___UInt256___arrayFirstIndex_x____x__3_ = r""" +Inline___UInt256___arrayFirstIndex_x____x__3__ = r""" arrayFirstIndex(lambda(tuple(x), equals(x, 3)), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___arrayFirstIndex_x____x__3_ = r""" +Table___UInt256___arrayFirstIndex_x____x__3__ = r""" a 1 """ -Inline___UInt256___hasAll__toUInt256__3____toUInt256__2____toUInt256__1_____ = r""" +Inline___UInt256___hasAll__toUInt256__3____toUInt256__2____toUInt256__1______ = r""" hasAll(array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\')), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___hasAll__toUInt256__3____toUInt256__2____toUInt256__1_____ = r""" +Table___UInt256___hasAll__toUInt256__3____toUInt256__2____toUInt256__1______ = r""" a 1 """ -Inline___UInt256___hasAny__toUInt256__2____toUInt256__1_____ = r""" +Inline___UInt256___hasAny__toUInt256__2____toUInt256__1______ = r""" hasAny(array(toUInt256(\'2\'), toUInt256(\'1\')), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 1 """ -Table___UInt256___hasAny__toUInt256__2____toUInt256__1_____ = r""" +Table___UInt256___hasAny__toUInt256__2____toUInt256__1______ = r""" a 1 """ -Inline___UInt256___hasSubstr__toUInt256__2____toUInt256__1_____ = r""" +Inline___UInt256___hasSubstr__toUInt256__2____toUInt256__1______ = r""" hasSubstr(array(toUInt256(\'2\'), toUInt256(\'1\')), array(toUInt256(\'3\'), toUInt256(\'2\'), toUInt256(\'1\'))) 0 """ -Table___UInt256___hasSubstr__toUInt256__2____toUInt256__1_____ = r""" +Table___UInt256___hasSubstr__toUInt256__2____toUInt256__1______ = r""" a 0 """ -Table___UInt256___arrayDifference_ = r""" +Table___UInt256___arrayDifference__ = r""" a """ -Table___UInt256___arrayCumSum_ = r""" +Table___UInt256___arrayCumSum__ = r""" a """ -Table___UInt256___arrayCumSumNonNegative_ = r""" +Table___UInt256___arrayCumSumNonNegative__ = r""" a """ @@ -2644,375 +2644,375 @@ a [1,2] """ -Inline___Decimal256_0____arrayPopBack_ = r""" +Inline___Decimal256_0____arrayPopBack__ = r""" arrayPopBack(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2] """ -Table___Decimal256_0____arrayPopBack_ = r""" +Table___Decimal256_0____arrayPopBack__ = r""" a [3,2] """ -Inline___Decimal256_0____arrayPopFront_ = r""" +Inline___Decimal256_0____arrayPopFront__ = r""" arrayPopFront(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [2,1] """ -Table___Decimal256_0____arrayPopFront_ = r""" +Table___Decimal256_0____arrayPopFront__ = r""" a [2,1] """ -Inline___Decimal256_0____arraySort_ = r""" +Inline___Decimal256_0____arraySort__ = r""" arraySort(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,2,3] """ -Table___Decimal256_0____arraySort_ = r""" +Table___Decimal256_0____arraySort__ = r""" a [1,2,3] """ -Inline___Decimal256_0____arrayReverseSort_ = r""" +Inline___Decimal256_0____arrayReverseSort__ = r""" arrayReverseSort(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2,1] """ -Table___Decimal256_0____arrayReverseSort_ = r""" +Table___Decimal256_0____arrayReverseSort__ = r""" a [3,2,1] """ -Inline___Decimal256_0____arrayDistinct_ = r""" +Inline___Decimal256_0____arrayDistinct__ = r""" arrayDistinct(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2,1] """ -Table___Decimal256_0____arrayDistinct_ = r""" +Table___Decimal256_0____arrayDistinct__ = r""" a [3,2,1] """ -Inline___Decimal256_0____arrayEnumerate_ = r""" +Inline___Decimal256_0____arrayEnumerate__ = r""" arrayEnumerate(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,2,3] """ -Table___Decimal256_0____arrayEnumerate_ = r""" +Table___Decimal256_0____arrayEnumerate__ = r""" a [1,2,3] """ -Inline___Decimal256_0____arrayEnumerateDense_ = r""" +Inline___Decimal256_0____arrayEnumerateDense__ = r""" arrayEnumerateDense(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,2,3] """ -Table___Decimal256_0____arrayEnumerateDense_ = r""" +Table___Decimal256_0____arrayEnumerateDense__ = r""" a [1,2,3] """ -Inline___Decimal256_0____arrayEnumerateUniq_ = r""" +Inline___Decimal256_0____arrayEnumerateUniq__ = r""" arrayEnumerateUniq(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,1,1] """ -Table___Decimal256_0____arrayEnumerateUniq_ = r""" +Table___Decimal256_0____arrayEnumerateUniq__ = r""" a [1,1,1] """ -Inline___Decimal256_0____arrayReverse_ = r""" +Inline___Decimal256_0____arrayReverse__ = r""" arrayReverse(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,2,3] """ -Table___Decimal256_0____arrayReverse_ = r""" +Table___Decimal256_0____arrayReverse__ = r""" a [1,2,3] """ -Inline___Decimal256_0____reverse_ = r""" +Inline___Decimal256_0____reverse__ = r""" reverse(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1,2,3] """ -Table___Decimal256_0____reverse_ = r""" +Table___Decimal256_0____reverse__ = r""" a [1,2,3] """ -Inline___Decimal256_0____arrayFlatten_ = r""" +Inline___Decimal256_0____arrayFlatten__ = r""" arrayFlatten(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2,1] """ -Table___Decimal256_0____arrayFlatten_ = r""" +Table___Decimal256_0____arrayFlatten__ = r""" a [3,2,1] """ -Inline___Decimal256_0____arrayCompact_ = r""" +Inline___Decimal256_0____arrayCompact__ = r""" arrayCompact(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2,1] """ -Table___Decimal256_0____arrayCompact_ = r""" +Table___Decimal256_0____arrayCompact__ = r""" a [3,2,1] """ -Inline___Decimal256_0____arrayReduceInRanges__sum_____1__5___ = r""" +Inline___Decimal256_0____arrayReduceInRanges__sum_____1__5____ = r""" arrayReduceInRanges(\'sum\', array(tuple(1, 5)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [6] """ -Table___Decimal256_0____arrayReduceInRanges__sum_____1__5___ = r""" +Table___Decimal256_0____arrayReduceInRanges__sum_____1__5____ = r""" a [6] """ -Inline___Decimal256_0____arrayMap_x_____x___2__ = r""" +Inline___Decimal256_0____arrayMap_x_____x___2___ = r""" arrayMap(lambda(tuple(x), plus(x, 2)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [5,4,3] """ -Table___Decimal256_0____arrayMap_x_____x___2__ = r""" +Table___Decimal256_0____arrayMap_x_____x___2___ = r""" a [5,4,3] """ -Inline___Decimal256_0____arrayFill_x____x_3_ = r""" +Inline___Decimal256_0____arrayFill_x____x_3__ = r""" arrayFill(lambda(tuple(x), equals(x, 3)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,3,3] """ -Table___Decimal256_0____arrayFill_x____x_3_ = r""" +Table___Decimal256_0____arrayFill_x____x_3__ = r""" a [3,3,3] """ -Inline___Decimal256_0____arrayReverseFill_x____x_3_ = r""" +Inline___Decimal256_0____arrayReverseFill_x____x_3__ = r""" arrayReverseFill(lambda(tuple(x), equals(x, 3)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,1,1] """ -Table___Decimal256_0____arrayReverseFill_x____x_3_ = r""" +Table___Decimal256_0____arrayReverseFill_x____x_3__ = r""" a [3,1,1] """ -Inline___Decimal256_0____arrayConcat__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0___ = r""" +Inline___Decimal256_0____arrayConcat__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0____ = r""" arrayConcat(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [3,2,1,3,2,1] """ -Table___Decimal256_0____arrayConcat__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0___ = r""" +Table___Decimal256_0____arrayConcat__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0____ = r""" a [3,2,1,3,2,1] """ -Inline___Decimal256_0____arrayFilter_x____x____1__ = r""" +Inline___Decimal256_0____arrayFilter_x____x____1___ = r""" arrayFilter(lambda(tuple(x), equals(x, 1)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [1] """ -Table___Decimal256_0____arrayFilter_x____x____1__ = r""" +Table___Decimal256_0____arrayFilter_x____x____1___ = r""" a [1] """ -Inline___Decimal256_0____arraySplit__x__y_____x_y___0__0__0__ = r""" +Inline___Decimal256_0____arraySplit__x__y_____x_y___0__0__0___ = r""" arraySplit(lambda(tuple(x, y), equals(x, y)), [0, 0, 0], array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) [[0,0,0]] """ -Table___Decimal256_0____arraySplit__x__y_____x_y___0__0__0__ = r""" +Table___Decimal256_0____arraySplit__x__y_____x_y___0__0__0___ = r""" a [[0,0,0]] """ -Inline___Decimal256_0____arrayZip__toDecimal256__1__0___ = r""" +Inline___Decimal256_0____arrayZip__toDecimal256__1__0____ = r""" arrayZip(array(toDecimal256(\'1\', 0)), array(toDecimal256(\'3\', 0))) [(1,3)] """ -Table___Decimal256_0____arrayZip__toDecimal256__1__0___ = r""" +Table___Decimal256_0____arrayZip__toDecimal256__1__0____ = r""" a [(1,1)] """ -Inline___Decimal256_0____empty_ = r""" +Inline___Decimal256_0____empty__ = r""" empty(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 0 """ -Table___Decimal256_0____empty_ = r""" +Table___Decimal256_0____empty__ = r""" a 0 """ -Inline___Decimal256_0____notEmpty_ = r""" +Inline___Decimal256_0____notEmpty__ = r""" notEmpty(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____notEmpty_ = r""" +Table___Decimal256_0____notEmpty__ = r""" a 1 """ -Inline___Decimal256_0____length_ = r""" +Inline___Decimal256_0____length__ = r""" length(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 3 """ -Table___Decimal256_0____length_ = r""" +Table___Decimal256_0____length__ = r""" a 3 """ -Inline___Decimal256_0____arrayCount_x____x____1__ = r""" +Inline___Decimal256_0____arrayCount_x____x____1___ = r""" arrayCount(lambda(tuple(x), equals(x, 1)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____arrayCount_x____x____1__ = r""" +Table___Decimal256_0____arrayCount_x____x____1___ = r""" a 1 """ -Inline___Decimal256_0____arrayUniq_ = r""" +Inline___Decimal256_0____arrayUniq__ = r""" arrayUniq(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 3 """ -Table___Decimal256_0____arrayUniq_ = r""" +Table___Decimal256_0____arrayUniq__ = r""" a 3 """ -Inline___Decimal256_0____arrayJoin_ = r""" +Inline___Decimal256_0____arrayJoin__ = r""" arrayJoin(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 3 2 1 """ -Table___Decimal256_0____arrayJoin_ = r""" +Table___Decimal256_0____arrayJoin__ = r""" a 1 2 3 """ -Inline___Decimal256_0____arrayExists_x____x__1_ = r""" +Inline___Decimal256_0____arrayExists_x____x__1__ = r""" arrayExists(lambda(tuple(x), equals(x, 1)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____arrayExists_x____x__1_ = r""" +Table___Decimal256_0____arrayExists_x____x__1__ = r""" a 1 """ -Inline___Decimal256_0____arrayAll_x____x__1_ = r""" +Inline___Decimal256_0____arrayAll_x____x__1__ = r""" arrayAll(lambda(tuple(x), equals(x, 1)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 0 """ -Table___Decimal256_0____arrayAll_x____x__1_ = r""" +Table___Decimal256_0____arrayAll_x____x__1__ = r""" a 0 """ -Table___Decimal256_0____arrayMin_ = r""" +Table___Decimal256_0____arrayMin__ = r""" a """ -Table___Decimal256_0____arrayMax_ = r""" +Table___Decimal256_0____arrayMax__ = r""" a """ -Table___Decimal256_0____arraySum_ = r""" +Table___Decimal256_0____arraySum__ = r""" a """ -Table___Decimal256_0____arrayAvg_ = r""" +Table___Decimal256_0____arrayAvg__ = r""" a """ -Inline___Decimal256_0____arrayReduce__max___ = r""" +Inline___Decimal256_0____arrayReduce__max____ = r""" arrayReduce(\'max\', array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 3 """ -Table___Decimal256_0____arrayReduce__max___ = r""" +Table___Decimal256_0____arrayReduce__max____ = r""" a 3 """ -Inline___Decimal256_0____arrayFirst_x____x__3_ = r""" +Inline___Decimal256_0____arrayFirst_x____x__3__ = r""" arrayFirst(lambda(tuple(x), equals(x, 3)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 3 """ -Table___Decimal256_0____arrayFirst_x____x__3_ = r""" +Table___Decimal256_0____arrayFirst_x____x__3__ = r""" a 3 """ -Inline___Decimal256_0____arrayFirstIndex_x____x__3_ = r""" +Inline___Decimal256_0____arrayFirstIndex_x____x__3__ = r""" arrayFirstIndex(lambda(tuple(x), equals(x, 3)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____arrayFirstIndex_x____x__3_ = r""" +Table___Decimal256_0____arrayFirstIndex_x____x__3__ = r""" a 1 """ -Inline___Decimal256_0____hasAll__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0____ = r""" +Inline___Decimal256_0____hasAll__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0_____ = r""" hasAll(array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____hasAll__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0____ = r""" +Table___Decimal256_0____hasAll__toDecimal256__3__0___toDecimal256__2__0___toDecimal256__1__0_____ = r""" a 1 """ -Inline___Decimal256_0____hasAny__toDecimal256__2__0___toDecimal256__1__0____ = r""" +Inline___Decimal256_0____hasAny__toDecimal256__2__0___toDecimal256__1__0_____ = r""" hasAny(array(toDecimal256(\'2\', 0), toDecimal256(\'1\', 0)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 1 """ -Table___Decimal256_0____hasAny__toDecimal256__2__0___toDecimal256__1__0____ = r""" +Table___Decimal256_0____hasAny__toDecimal256__2__0___toDecimal256__1__0_____ = r""" a 1 """ -Inline___Decimal256_0____hasSubstr__toDecimal256__2__0___toDecimal256__1__0____ = r""" +Inline___Decimal256_0____hasSubstr__toDecimal256__2__0___toDecimal256__1__0_____ = r""" hasSubstr(array(toDecimal256(\'2\', 0), toDecimal256(\'1\', 0)), array(toDecimal256(\'3\', 0), toDecimal256(\'2\', 0), toDecimal256(\'1\', 0))) 0 """ -Table___Decimal256_0____hasSubstr__toDecimal256__2__0___toDecimal256__1__0____ = r""" +Table___Decimal256_0____hasSubstr__toDecimal256__2__0___toDecimal256__1__0_____ = r""" a 0 """ -Table___Decimal256_0____arrayDifference_ = r""" +Table___Decimal256_0____arrayDifference__ = r""" a """ -Table___Decimal256_0____arrayCumSum_ = r""" +Table___Decimal256_0____arrayCumSum__ = r""" a """ -Table___Decimal256_0____arrayCumSumNonNegative_ = r""" +Table___Decimal256_0____arrayCumSumNonNegative__ = r""" a """ @@ -3509,543 +3509,543 @@ a """ I_check_exp__with_Int128_using_max_and_min = r""" -exp(toInt128(\'170141183460469231731687303715884105727\')) exp(toInt128(\'-170141183460469231731687303715884105728\')) +round(exp(toInt128(\'170141183460469231731687303715884105727\')), 7) round(exp(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf 0 """ I_check_exp__with_Int256_using_max_and_min = r""" -exp(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) exp(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(exp(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(exp(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf 0 """ I_check_exp__with_UInt128_using_max_and_min = r""" -exp(toUInt128(\'340282366920938463463374607431768211455\')) exp(toUInt128(\'0\')) +round(exp(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(exp(toUInt128(\'0\')), 7) inf 1 """ I_check_exp__with_UInt256_using_max_and_min = r""" -exp(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) exp(toUInt256(\'0\')) +round(exp(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(exp(toUInt256(\'0\')), 7) inf 1 """ I_check_log__with_Int128_using_max_and_min = r""" -log(toInt128(\'170141183460469231731687303715884105727\')) log(toInt128(\'-170141183460469231731687303715884105728\')) -88.02969193111305 nan +round(log(toInt128(\'170141183460469231731687303715884105727\')), 7) round(log(toInt128(\'-170141183460469231731687303715884105728\')), 7) +88.0296919 nan """ I_check_log__with_Int256_using_max_and_min = r""" -log(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) log(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -176.75253104278605 nan +round(log(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(log(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +176.752531 nan """ I_check_log__with_UInt128_using_max_and_min = r""" -log(toUInt128(\'340282366920938463463374607431768211455\')) log(toUInt128(\'0\')) -88.722839111673 -inf +round(log(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(log(toUInt128(\'0\')), 7) +88.7228391 -inf """ I_check_log__with_UInt256_using_max_and_min = r""" -log(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) log(toUInt256(\'0\')) -177.445678223346 -inf +round(log(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(log(toUInt256(\'0\')), 7) +177.4456782 -inf """ I_check_ln__with_Int128_using_max_and_min = r""" -log(toInt128(\'170141183460469231731687303715884105727\')) log(toInt128(\'-170141183460469231731687303715884105728\')) -88.02969193111305 nan +round(log(toInt128(\'170141183460469231731687303715884105727\')), 7) round(log(toInt128(\'-170141183460469231731687303715884105728\')), 7) +88.0296919 nan """ I_check_ln__with_Int256_using_max_and_min = r""" -log(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) log(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -176.75253104278605 nan +round(log(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(log(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +176.752531 nan """ I_check_ln__with_UInt128_using_max_and_min = r""" -log(toUInt128(\'340282366920938463463374607431768211455\')) log(toUInt128(\'0\')) -88.722839111673 -inf +round(log(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(log(toUInt128(\'0\')), 7) +88.7228391 -inf """ I_check_ln__with_UInt256_using_max_and_min = r""" -log(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) log(toUInt256(\'0\')) -177.445678223346 -inf +round(log(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(log(toUInt256(\'0\')), 7) +177.4456782 -inf """ I_check_exp2__with_Int128_using_max_and_min = r""" -exp2(toInt128(\'170141183460469231731687303715884105727\')) exp2(toInt128(\'-170141183460469231731687303715884105728\')) +round(exp2(toInt128(\'170141183460469231731687303715884105727\')), 7) round(exp2(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf 0 """ I_check_exp2__with_Int256_using_max_and_min = r""" -exp2(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) exp2(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(exp2(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(exp2(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf 0 """ I_check_exp2__with_UInt128_using_max_and_min = r""" -exp2(toUInt128(\'340282366920938463463374607431768211455\')) exp2(toUInt128(\'0\')) +round(exp2(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(exp2(toUInt128(\'0\')), 7) inf 1 """ I_check_exp2__with_UInt256_using_max_and_min = r""" -exp2(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) exp2(toUInt256(\'0\')) +round(exp2(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(exp2(toUInt256(\'0\')), 7) inf 1 """ I_check_log2__with_Int128_using_max_and_min = r""" -log2(toInt128(\'170141183460469231731687303715884105727\')) log2(toInt128(\'-170141183460469231731687303715884105728\')) +round(log2(toInt128(\'170141183460469231731687303715884105727\')), 7) round(log2(toInt128(\'-170141183460469231731687303715884105728\')), 7) 127 nan """ I_check_log2__with_Int256_using_max_and_min = r""" -log2(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) log2(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(log2(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(log2(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 255 nan """ I_check_log2__with_UInt128_using_max_and_min = r""" -log2(toUInt128(\'340282366920938463463374607431768211455\')) log2(toUInt128(\'0\')) +round(log2(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(log2(toUInt128(\'0\')), 7) 128 -inf """ I_check_log2__with_UInt256_using_max_and_min = r""" -log2(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) log2(toUInt256(\'0\')) +round(log2(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(log2(toUInt256(\'0\')), 7) 256 -inf """ I_check_exp10__with_Int128_using_max_and_min = r""" -exp10(toInt128(\'170141183460469231731687303715884105727\')) exp10(toInt128(\'-170141183460469231731687303715884105728\')) +round(exp10(toInt128(\'170141183460469231731687303715884105727\')), 7) round(exp10(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf 0 """ I_check_exp10__with_Int256_using_max_and_min = r""" -exp10(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) exp10(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(exp10(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(exp10(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf 0 """ I_check_exp10__with_UInt128_using_max_and_min = r""" -exp10(toUInt128(\'340282366920938463463374607431768211455\')) exp10(toUInt128(\'0\')) +round(exp10(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(exp10(toUInt128(\'0\')), 7) inf 1 """ I_check_exp10__with_UInt256_using_max_and_min = r""" -exp10(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) exp10(toUInt256(\'0\')) +round(exp10(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(exp10(toUInt256(\'0\')), 7) inf 1 """ I_check_log10__with_Int128_using_max_and_min = r""" -log10(toInt128(\'170141183460469231731687303715884105727\')) log10(toInt128(\'-170141183460469231731687303715884105728\')) -38.23080944932561 nan +round(log10(toInt128(\'170141183460469231731687303715884105727\')), 7) round(log10(toInt128(\'-170141183460469231731687303715884105728\')), 7) +38.2308094 nan """ I_check_log10__with_Int256_using_max_and_min = r""" -log10(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) log10(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -76.7626488943152 nan +round(log10(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(log10(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +76.7626489 nan """ I_check_log10__with_UInt128_using_max_and_min = r""" -log10(toUInt128(\'340282366920938463463374607431768211455\')) log10(toUInt128(\'0\')) -38.53183944498959 -inf +round(log10(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(log10(toUInt128(\'0\')), 7) +38.5318394 -inf """ I_check_log10__with_UInt256_using_max_and_min = r""" -log10(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) log10(toUInt256(\'0\')) -77.06367888997919 -inf +round(log10(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(log10(toUInt256(\'0\')), 7) +77.0636789 -inf """ I_check_sqrt__with_Int128_using_max_and_min = r""" -sqrt(toInt128(\'170141183460469231731687303715884105727\')) sqrt(toInt128(\'-170141183460469231731687303715884105728\')) +round(sqrt(toInt128(\'170141183460469231731687303715884105727\')), 7) round(sqrt(toInt128(\'-170141183460469231731687303715884105728\')), 7) 13043817825332783000 nan """ I_check_sqrt__with_Int256_using_max_and_min = r""" -sqrt(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) sqrt(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(sqrt(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(sqrt(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 2.4061596916800453e38 nan """ I_check_sqrt__with_UInt128_using_max_and_min = r""" -sqrt(toUInt128(\'340282366920938463463374607431768211455\')) sqrt(toUInt128(\'0\')) +round(sqrt(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(sqrt(toUInt128(\'0\')), 7) 18446744073709552000 0 """ I_check_sqrt__with_UInt256_using_max_and_min = r""" -sqrt(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) sqrt(toUInt256(\'0\')) +round(sqrt(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(sqrt(toUInt256(\'0\')), 7) 3.402823669209385e38 0 """ I_check_cbrt__with_Int128_using_max_and_min = r""" -cbrt(toInt128(\'170141183460469231731687303715884105727\')) cbrt(toInt128(\'-170141183460469231731687303715884105728\')) +round(cbrt(toInt128(\'170141183460469231731687303715884105727\')), 7) round(cbrt(toInt128(\'-170141183460469231731687303715884105728\')), 7) 5541191377756.637 -5541191377756.637 """ I_check_cbrt__with_Int256_using_max_and_min = r""" -cbrt(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) cbrt(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(cbrt(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(cbrt(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 3.8685626227668134e25 -3.8685626227668134e25 """ I_check_cbrt__with_UInt128_using_max_and_min = r""" -cbrt(toUInt128(\'340282366920938463463374607431768211455\')) cbrt(toUInt128(\'0\')) +round(cbrt(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(cbrt(toUInt128(\'0\')), 7) 6981463658331.56 0 """ I_check_cbrt__with_UInt256_using_max_and_min = r""" -cbrt(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) cbrt(toUInt256(\'0\')) +round(cbrt(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(cbrt(toUInt256(\'0\')), 7) 4.874083481260429e25 0 """ I_check_erf__with_Int128_using_max_and_min = r""" -erf(toInt128(\'170141183460469231731687303715884105727\')) erf(toInt128(\'-170141183460469231731687303715884105728\')) +round(erf(toInt128(\'170141183460469231731687303715884105727\')), 7) round(erf(toInt128(\'-170141183460469231731687303715884105728\')), 7) 1 -1 """ I_check_erf__with_Int256_using_max_and_min = r""" -erf(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) erf(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(erf(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(erf(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 1 -1 """ I_check_erf__with_UInt128_using_max_and_min = r""" -erf(toUInt128(\'340282366920938463463374607431768211455\')) erf(toUInt128(\'0\')) +round(erf(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(erf(toUInt128(\'0\')), 7) 1 0 """ I_check_erf__with_UInt256_using_max_and_min = r""" -erf(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) erf(toUInt256(\'0\')) +round(erf(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(erf(toUInt256(\'0\')), 7) 1 0 """ I_check_erfc__with_Int128_using_max_and_min = r""" -erfc(toInt128(\'170141183460469231731687303715884105727\')) erfc(toInt128(\'-170141183460469231731687303715884105728\')) +round(erfc(toInt128(\'170141183460469231731687303715884105727\')), 7) round(erfc(toInt128(\'-170141183460469231731687303715884105728\')), 7) 0 2 """ I_check_erfc__with_Int256_using_max_and_min = r""" -erfc(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) erfc(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(erfc(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(erfc(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 0 2 """ I_check_erfc__with_UInt128_using_max_and_min = r""" -erfc(toUInt128(\'340282366920938463463374607431768211455\')) erfc(toUInt128(\'0\')) +round(erfc(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(erfc(toUInt128(\'0\')), 7) 0 1 """ I_check_erfc__with_UInt256_using_max_and_min = r""" -erfc(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) erfc(toUInt256(\'0\')) +round(erfc(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(erfc(toUInt256(\'0\')), 7) 0 1 """ I_check_lgamma__with_Int128_using_max_and_min = r""" -lgamma(toInt128(\'170141183460469231731687303715884105727\')) lgamma(toInt128(\'-170141183460469231731687303715884105728\')) +round(lgamma(toInt128(\'170141183460469231731687303715884105727\')), 7) round(lgamma(toInt128(\'-170141183460469231731687303715884105728\')), 7) 1.4807334781359624e40 -1.4807334781359624e40 """ I_check_lgamma__with_Int256_using_max_and_min = r""" -lgamma(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) lgamma(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(lgamma(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(lgamma(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 1.0175376379095233e79 -1.0175376379095233e79 """ I_check_lgamma__with_UInt128_using_max_and_min = r""" -lgamma(toUInt128(\'340282366920938463463374607431768211455\')) lgamma(toUInt128(\'0\')) +round(lgamma(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(lgamma(toUInt128(\'0\')), 7) 2.985053532594476e40 inf """ I_check_lgamma__with_UInt256_using_max_and_min = r""" -lgamma(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) lgamma(toUInt256(\'0\')) +round(lgamma(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(lgamma(toUInt256(\'0\')), 7) 2.0431013718376458e79 inf """ I_check_tgamma__with_Int128_using_max_and_min = r""" -tgamma(toInt128(\'170141183460469231731687303715884105727\')) tgamma(toInt128(\'-170141183460469231731687303715884105728\')) +round(tgamma(toInt128(\'170141183460469231731687303715884105727\')), 7) round(tgamma(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf nan """ I_check_tgamma__with_Int256_using_max_and_min = r""" -tgamma(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) tgamma(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(tgamma(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(tgamma(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf nan """ I_check_tgamma__with_UInt128_using_max_and_min = r""" -tgamma(toUInt128(\'340282366920938463463374607431768211455\')) tgamma(toUInt128(\'0\')) +round(tgamma(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(tgamma(toUInt128(\'0\')), 7) inf inf """ I_check_tgamma__with_UInt256_using_max_and_min = r""" -tgamma(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) tgamma(toUInt256(\'0\')) +round(tgamma(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(tgamma(toUInt256(\'0\')), 7) inf inf """ I_check_sin__with_Int128_using_max_and_min = r""" -sin(toInt128(\'170141183460469231731687303715884105727\')) sin(toInt128(\'-170141183460469231731687303715884105728\')) -0.6233855129558702 -0.6233855129558702 +round(sin(toInt128(\'170141183460469231731687303715884105727\')), 7) round(sin(toInt128(\'-170141183460469231731687303715884105728\')), 7) +0.6233855 -0.6233855 """ I_check_sin__with_Int256_using_max_and_min = r""" -sin(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) sin(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -0.9751222164851924 -0.9751222164851924 +round(sin(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(sin(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +0.9751222 -0.9751222 """ I_check_sin__with_UInt128_using_max_and_min = r""" -sin(toUInt128(\'340282366920938463463374607431768211455\')) sin(toUInt128(\'0\')) -0.9748685162860586 0 +round(sin(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(sin(toUInt128(\'0\')), 7) +0.9748685 0 """ I_check_sin__with_UInt256_using_max_and_min = r""" -sin(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) sin(toUInt256(\'0\')) -0.4323066100553458 0 +round(sin(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(sin(toUInt256(\'0\')), 7) +0.4323066 0 """ I_check_cos__with_Int128_using_max_and_min = r""" -cos(toInt128(\'170141183460469231731687303715884105727\')) cos(toInt128(\'-170141183460469231731687303715884105728\')) -0.78191463871496 0.78191463871496 +round(cos(toInt128(\'170141183460469231731687303715884105727\')), 7) round(cos(toInt128(\'-170141183460469231731687303715884105728\')), 7) +0.7819146 0.7819146 """ I_check_cos__with_Int256_using_max_and_min = r""" -cos(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) cos(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -0.22166791133812228 0.22166791133812228 +round(cos(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(cos(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +0.2216679 0.2216679 """ I_check_cos__with_UInt128_using_max_and_min = r""" -cos(toUInt128(\'340282366920938463463374607431768211455\')) cos(toUInt128(\'0\')) -0.22278100447349308 1 +round(cos(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(cos(toUInt128(\'0\')), 7) +0.222781 1 """ I_check_cos__with_UInt256_using_max_and_min = r""" -cos(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) cos(toUInt256(\'0\')) --0.9017266741659887 1 +round(cos(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(cos(toUInt256(\'0\')), 7) +-0.9017267 1 """ I_check_tan__with_Int128_using_max_and_min = r""" -tan(toInt128(\'170141183460469231731687303715884105727\')) tan(toInt128(\'-170141183460469231731687303715884105728\')) -0.7972552016424389 -0.7972552016424389 +round(tan(toInt128(\'170141183460469231731687303715884105727\')), 7) round(tan(toInt128(\'-170141183460469231731687303715884105728\')), 7) +0.7972552 -0.7972552 """ I_check_tan__with_Int256_using_max_and_min = r""" -tan(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) tan(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -4.399022892392326 -4.399022892392326 +round(tan(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(tan(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +4.3990229 -4.3990229 """ I_check_tan__with_UInt128_using_max_and_min = r""" -tan(toUInt128(\'340282366920938463463374607431768211455\')) tan(toUInt128(\'0\')) -4.375905022019283 0 +round(tan(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(tan(toUInt128(\'0\')), 7) +4.375905 0 """ I_check_tan__with_UInt256_using_max_and_min = r""" -tan(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) tan(toUInt256(\'0\')) --0.4794208959773628 0 +round(tan(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(tan(toUInt256(\'0\')), 7) +-0.4794209 0 """ I_check_asin__with_Int128_using_max_and_min = r""" -asin(toInt128(\'170141183460469231731687303715884105727\')) asin(toInt128(\'-170141183460469231731687303715884105728\')) +round(asin(toInt128(\'170141183460469231731687303715884105727\')), 7) round(asin(toInt128(\'-170141183460469231731687303715884105728\')), 7) nan nan """ I_check_asin__with_Int256_using_max_and_min = r""" -asin(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) asin(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(asin(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(asin(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) nan nan """ I_check_asin__with_UInt128_using_max_and_min = r""" -asin(toUInt128(\'340282366920938463463374607431768211455\')) asin(toUInt128(\'0\')) +round(asin(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(asin(toUInt128(\'0\')), 7) nan 0 """ I_check_asin__with_UInt256_using_max_and_min = r""" -asin(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) asin(toUInt256(\'0\')) +round(asin(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(asin(toUInt256(\'0\')), 7) nan 0 """ I_check_acos__with_Int128_using_max_and_min = r""" -acos(toInt128(\'170141183460469231731687303715884105727\')) acos(toInt128(\'-170141183460469231731687303715884105728\')) +round(acos(toInt128(\'170141183460469231731687303715884105727\')), 7) round(acos(toInt128(\'-170141183460469231731687303715884105728\')), 7) nan nan """ I_check_acos__with_Int256_using_max_and_min = r""" -acos(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) acos(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(acos(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(acos(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) nan nan """ I_check_acos__with_UInt128_using_max_and_min = r""" -acos(toUInt128(\'340282366920938463463374607431768211455\')) acos(toUInt128(\'0\')) -nan 1.5707963267948966 +round(acos(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(acos(toUInt128(\'0\')), 7) +nan 1.5707963 """ I_check_acos__with_UInt256_using_max_and_min = r""" -acos(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) acos(toUInt256(\'0\')) -nan 1.5707963267948966 +round(acos(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(acos(toUInt256(\'0\')), 7) +nan 1.5707963 """ I_check_atan__with_Int128_using_max_and_min = r""" -atan(toInt128(\'170141183460469231731687303715884105727\')) atan(toInt128(\'-170141183460469231731687303715884105728\')) -1.5707963267948966 -1.5707963267948966 +round(atan(toInt128(\'170141183460469231731687303715884105727\')), 7) round(atan(toInt128(\'-170141183460469231731687303715884105728\')), 7) +1.5707963 -1.5707963 """ I_check_atan__with_Int256_using_max_and_min = r""" -atan(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) atan(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -1.5707963267948966 -1.5707963267948966 +round(atan(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(atan(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +1.5707963 -1.5707963 """ I_check_atan__with_UInt128_using_max_and_min = r""" -atan(toUInt128(\'340282366920938463463374607431768211455\')) atan(toUInt128(\'0\')) -1.5707963267948966 0 +round(atan(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(atan(toUInt128(\'0\')), 7) +1.5707963 0 """ I_check_atan__with_UInt256_using_max_and_min = r""" -atan(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) atan(toUInt256(\'0\')) -1.5707963267948966 0 +round(atan(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(atan(toUInt256(\'0\')), 7) +1.5707963 0 """ I_check_cosh__with_Int128_using_max_and_min = r""" -cosh(toInt128(\'170141183460469231731687303715884105727\')) cosh(toInt128(\'-170141183460469231731687303715884105728\')) +round(cosh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(cosh(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf inf """ I_check_cosh__with_Int256_using_max_and_min = r""" -cosh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) cosh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(cosh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(cosh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf inf """ I_check_cosh__with_UInt128_using_max_and_min = r""" -cosh(toUInt128(\'340282366920938463463374607431768211455\')) cosh(toUInt128(\'0\')) +round(cosh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(cosh(toUInt128(\'0\')), 7) inf 1 """ I_check_cosh__with_UInt256_using_max_and_min = r""" -cosh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) cosh(toUInt256(\'0\')) +round(cosh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(cosh(toUInt256(\'0\')), 7) inf 1 """ I_check_acosh__with_Int128_using_max_and_min = r""" -acosh(toInt128(\'170141183460469231731687303715884105727\')) acosh(toInt128(\'-170141183460469231731687303715884105728\')) -88.722839111673 nan +round(acosh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(acosh(toInt128(\'-170141183460469231731687303715884105728\')), 7) +88.7228391 nan """ I_check_acosh__with_Int256_using_max_and_min = r""" -acosh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) acosh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -177.445678223346 nan +round(acosh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(acosh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +177.4456782 nan """ I_check_acosh__with_UInt128_using_max_and_min = r""" -acosh(toUInt128(\'340282366920938463463374607431768211455\')) acosh(toUInt128(\'0\')) -89.41598629223294 nan +round(acosh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(acosh(toUInt128(\'0\')), 7) +89.4159863 nan """ I_check_acosh__with_UInt256_using_max_and_min = r""" -acosh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) acosh(toUInt256(\'0\')) -178.13882540390594 nan +round(acosh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(acosh(toUInt256(\'0\')), 7) +178.1388254 nan """ I_check_sinh__with_Int128_using_max_and_min = r""" -sinh(toInt128(\'170141183460469231731687303715884105727\')) sinh(toInt128(\'-170141183460469231731687303715884105728\')) +round(sinh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(sinh(toInt128(\'-170141183460469231731687303715884105728\')), 7) inf -inf """ I_check_sinh__with_Int256_using_max_and_min = r""" -sinh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) sinh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(sinh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(sinh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) inf -inf """ I_check_sinh__with_UInt128_using_max_and_min = r""" -sinh(toUInt128(\'340282366920938463463374607431768211455\')) sinh(toUInt128(\'0\')) +round(sinh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(sinh(toUInt128(\'0\')), 7) inf 0 """ I_check_sinh__with_UInt256_using_max_and_min = r""" -sinh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) sinh(toUInt256(\'0\')) +round(sinh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(sinh(toUInt256(\'0\')), 7) inf 0 """ I_check_asinh__with_Int128_using_max_and_min = r""" -asinh(toInt128(\'170141183460469231731687303715884105727\')) asinh(toInt128(\'-170141183460469231731687303715884105728\')) -88.722839111673 -88.722839111673 +round(asinh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(asinh(toInt128(\'-170141183460469231731687303715884105728\')), 7) +88.7228391 -88.7228391 """ I_check_asinh__with_Int256_using_max_and_min = r""" -asinh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) asinh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -177.445678223346 -177.445678223346 +round(asinh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(asinh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +177.4456782 -177.4456782 """ I_check_asinh__with_UInt128_using_max_and_min = r""" -asinh(toUInt128(\'340282366920938463463374607431768211455\')) asinh(toUInt128(\'0\')) -89.41598629223294 0 +round(asinh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(asinh(toUInt128(\'0\')), 7) +89.4159863 0 """ I_check_asinh__with_UInt256_using_max_and_min = r""" -asinh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) asinh(toUInt256(\'0\')) -178.13882540390594 0 +round(asinh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(asinh(toUInt256(\'0\')), 7) +178.1388254 0 """ I_check_tanh__with_Int128_using_max_and_min = r""" -tanh(toInt128(\'170141183460469231731687303715884105727\')) tanh(toInt128(\'-170141183460469231731687303715884105728\')) +round(tanh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(tanh(toInt128(\'-170141183460469231731687303715884105728\')), 7) 1 -1 """ I_check_tanh__with_Int256_using_max_and_min = r""" -tanh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) tanh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(tanh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(tanh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) 1 -1 """ I_check_tanh__with_UInt128_using_max_and_min = r""" -tanh(toUInt128(\'340282366920938463463374607431768211455\')) tanh(toUInt128(\'0\')) +round(tanh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(tanh(toUInt128(\'0\')), 7) 1 0 """ I_check_tanh__with_UInt256_using_max_and_min = r""" -tanh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) tanh(toUInt256(\'0\')) +round(tanh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(tanh(toUInt256(\'0\')), 7) 1 0 """ I_check_atanh__with_Int128_using_max_and_min = r""" -atanh(toInt128(\'170141183460469231731687303715884105727\')) atanh(toInt128(\'-170141183460469231731687303715884105728\')) +round(atanh(toInt128(\'170141183460469231731687303715884105727\')), 7) round(atanh(toInt128(\'-170141183460469231731687303715884105728\')), 7) nan nan """ I_check_atanh__with_Int256_using_max_and_min = r""" -atanh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) atanh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) +round(atanh(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(atanh(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) nan nan """ I_check_atanh__with_UInt128_using_max_and_min = r""" -atanh(toUInt128(\'340282366920938463463374607431768211455\')) atanh(toUInt128(\'0\')) +round(atanh(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(atanh(toUInt128(\'0\')), 7) nan 0 """ I_check_atanh__with_UInt256_using_max_and_min = r""" -atanh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) atanh(toUInt256(\'0\')) +round(atanh(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(atanh(toUInt256(\'0\')), 7) nan 0 """ I_check_log1p__with_Int128_using_max_and_min = r""" -log1p(toInt128(\'170141183460469231731687303715884105727\')) log1p(toInt128(\'-170141183460469231731687303715884105728\')) -88.02969193111305 nan +round(log1p(toInt128(\'170141183460469231731687303715884105727\')), 7) round(log1p(toInt128(\'-170141183460469231731687303715884105728\')), 7) +88.0296919 nan """ I_check_log1p__with_Int256_using_max_and_min = r""" -log1p(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) log1p(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -176.75253104278605 nan +round(log1p(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(log1p(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +176.752531 nan """ I_check_log1p__with_UInt128_using_max_and_min = r""" -log1p(toUInt128(\'340282366920938463463374607431768211455\')) log1p(toUInt128(\'0\')) -88.722839111673 0 +round(log1p(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(log1p(toUInt128(\'0\')), 7) +88.7228391 0 """ I_check_log1p__with_UInt256_using_max_and_min = r""" -log1p(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) log1p(toUInt256(\'0\')) -177.445678223346 0 +round(log1p(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(log1p(toUInt256(\'0\')), 7) +177.4456782 0 """ I_check_sign__with_Int128_using_max_and_min = r""" -sign(toInt128(\'170141183460469231731687303715884105727\')) sign(toInt128(\'-170141183460469231731687303715884105728\')) -1 -1 +round(sign(toInt128(\'170141183460469231731687303715884105727\')), 7) round(sign(toInt128(\'-170141183460469231731687303715884105728\')), 7) +0 0 """ I_check_sign__with_Int256_using_max_and_min = r""" -sign(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')) sign(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')) -1 -1 +round(sign(toInt256(\'57896044618658097711785492504343953926634992332820282019728792003956564819967\')), 7) round(sign(toInt256(\'-57896044618658097711785492504343953926634992332820282019728792003956564819968\')), 7) +0 0 """ I_check_sign__with_UInt128_using_max_and_min = r""" -sign(toUInt128(\'340282366920938463463374607431768211455\')) sign(toUInt128(\'0\')) -1 0 +round(sign(toUInt128(\'340282366920938463463374607431768211455\')), 7) round(sign(toUInt128(\'0\')), 7) +0 0 """ I_check_sign__with_UInt256_using_max_and_min = r""" -sign(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')) sign(toUInt256(\'0\')) -1 0 +round(sign(toUInt256(\'115792089237316195423570985008687907853269984665640564039457584007913129639935\')), 7) round(sign(toUInt256(\'0\')), 7) +0 0 """ I_check_the_outputs_of_exp__with_Int128 = r""" @@ -4805,138 +4805,138 @@ a """ I_check_exp__with_Decimal256_using_max_and_min = r""" -exp(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) exp(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(exp(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(exp(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf 0 """ I_check_log__with_Decimal256_using_max_and_min = r""" -log(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) log(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -172.69388197463743 nan +round(log(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(log(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +172.693882 nan """ I_check_ln__with_Decimal256_using_max_and_min = r""" -log(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) log(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -172.69388197463743 nan +round(log(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(log(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +172.693882 nan """ I_check_exp2__with_Decimal256_using_max_and_min = r""" -exp2(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) exp2(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(exp2(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(exp2(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf 0 """ I_check_log2__with_Decimal256_using_max_and_min = r""" -log2(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) log2(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -249.14460711655218 nan +round(log2(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(log2(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +249.1446071 nan """ I_check_exp10__with_Decimal256_using_max_and_min = r""" -exp10(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) exp10(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(exp10(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(exp10(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf 0 """ I_check_log10__with_Decimal256_using_max_and_min = r""" -log10(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) log10(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(log10(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(log10(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 75 nan """ I_check_sqrt__with_Decimal256_using_max_and_min = r""" -sqrt(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) sqrt(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(sqrt(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(sqrt(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 3.1622776601683794e37 nan """ I_check_cbrt__with_Decimal256_using_max_and_min = r""" -cbrt(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) cbrt(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(cbrt(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(cbrt(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 1e25 -1e25 """ I_check_erf__with_Decimal256_using_max_and_min = r""" -erf(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) erf(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(erf(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(erf(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 1 -1 """ I_check_erfc__with_Decimal256_using_max_and_min = r""" -erfc(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) erfc(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(erfc(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(erfc(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 0 2 """ I_check_lgamma__with_Decimal256_using_max_and_min = r""" -lgamma(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) lgamma(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(lgamma(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(lgamma(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 1.7169388197455342e77 -1.7169388197455342e77 """ I_check_tgamma__with_Decimal256_using_max_and_min = r""" -tgamma(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) tgamma(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(tgamma(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(tgamma(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf nan """ I_check_sin__with_Decimal256_using_max_and_min = r""" -sin(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) sin(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -0.66339975236386 -0.66339975236386 +round(sin(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(sin(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +0.6633998 -0.6633998 """ I_check_cos__with_Decimal256_using_max_and_min = r""" -cos(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) cos(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) --0.7482651726250322 -0.7482651726250322 +round(cos(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(cos(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +-0.7482652 -0.7482652 """ I_check_tan__with_Decimal256_using_max_and_min = r""" -tan(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) tan(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) --0.8865837628611647 0.8865837628611647 +round(tan(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(tan(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +-0.8865838 0.8865838 """ I_check_asin__with_Decimal256_using_max_and_min = r""" -asin(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) asin(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(asin(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(asin(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) nan nan """ I_check_acos__with_Decimal256_using_max_and_min = r""" -acos(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) acos(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(acos(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(acos(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) nan nan """ I_check_atan__with_Decimal256_using_max_and_min = r""" -atan(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) atan(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -1.5707963267948966 -1.5707963267948966 +round(atan(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(atan(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +1.5707963 -1.5707963 """ I_check_cosh__with_Decimal256_using_max_and_min = r""" -cosh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) cosh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(cosh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(cosh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf inf """ I_check_acosh__with_Decimal256_using_max_and_min = r""" -acosh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) acosh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -173.38702915511337 nan +round(acosh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(acosh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +173.3870292 nan """ I_check_sinh__with_Decimal256_using_max_and_min = r""" -sinh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) sinh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(sinh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(sinh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) inf -inf """ I_check_asinh__with_Decimal256_using_max_and_min = r""" -asinh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) asinh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -173.38702915511337 -173.38702915511337 +round(asinh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(asinh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +173.3870292 -173.3870292 """ I_check_tanh__with_Decimal256_using_max_and_min = r""" -tanh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) tanh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(tanh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(tanh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) 1 -1 """ I_check_atanh__with_Decimal256_using_max_and_min = r""" -atanh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) atanh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) +round(atanh(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(atanh(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) nan nan """ I_check_log1p__with_Decimal256_using_max_and_min = r""" -log1p(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) log1p(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -172.69388197455342 nan +round(log1p(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(log1p(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +172.693882 nan """ I_check_sign__with_Decimal256_using_max_and_min = r""" -sign(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) sign(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)) -1 -1 +round(sign(toDecimal256(\'1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) round(sign(toDecimal256(\'-1000000000000000000000000000000000000000000000000000000000000000000000000000\', 0)), 7) +0 0 """ I_check_the_outputs_of_exp__with_Decimal256 = r""" diff --git a/tests/testflows/extended_precision_data_types/tests/arithmetic.py b/tests/testflows/extended_precision_data_types/tests/arithmetic.py index d939569d5a7..db15e6fcfab 100644 --- a/tests/testflows/extended_precision_data_types/tests/arithmetic.py +++ b/tests/testflows/extended_precision_data_types/tests/arithmetic.py @@ -63,7 +63,7 @@ def inline_check(self, arithmetic_func, expected_result, int_type, min, max, nod with When(f"I check {arithmetic_func} with {int_type} max and min value"): execute_query(f""" - SELECT {arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), {arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1)) + SELECT round({arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), 7), round({arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1)), 7) """) @TestOutline @@ -95,7 +95,7 @@ def table_check(self, arithmetic_func, expected_result, int_type, min, max, node else: with When(f"I insert {arithmetic_func} with {int_type} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(1), to{int_type}(1))") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(1), to{int_type}(1)), 7)") with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output @@ -125,7 +125,7 @@ def table_check(self, arithmetic_func, expected_result, int_type, min, max, node for value in [min, max]: with When(f"I insert {arithmetic_func} with {int_type} {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1))") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1)), 7)") with Then(f"I check the table output of {arithmetic_func} with {int_type}"): execute_query(f""" @@ -191,7 +191,7 @@ def table_check_dec(self, arithmetic_func, expected_result, node=None): else: with When(f"I insert {arithmetic_func} with toDecimal256 into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0)), 7)") with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index 6efa122ffdf..d1a5171d00a 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -39,12 +39,12 @@ def array_func(self, data_type, node=None): f'arrayConcat([{to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}],', 'arrayFilter(x -> x == 1, ']: - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): execute_query(f""" SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) """) - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = f'Array({data_type})') @@ -58,12 +58,12 @@ def array_func(self, data_type, node=None): for func in ['arraySplit((x, y) -> x=y, [0, 0, 0],']: - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): execute_query(f""" SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) """) - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = f'Array(Array({data_type}))') @@ -77,12 +77,12 @@ def array_func(self, data_type, node=None): for func in [f'arrayZip([{to_data_type(data_type,1)}],']: - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): execute_query(f""" SELECT {func}array({to_data_type(data_type,3)})) """) - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = f'Array(Tuple({data_type}, {data_type}))') @@ -115,11 +115,11 @@ def array_func(self, data_type, node=None): if func in ['arrayMin(','arrayMax(','arraySum(', 'arrayAvg('] and data_type in ['Decimal256(0)']: - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): node.query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", exitcode = 44, message = 'Exception:') - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = data_type) @@ -134,13 +134,13 @@ def array_func(self, data_type, node=None): else: - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): execute_query(f""" SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) """) - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = data_type) @@ -161,11 +161,11 @@ def array_func(self, data_type, node=None): else: exitcode = 43 - with Scenario(f"Inline - {data_type} - {func}"): + with Scenario(f"Inline - {data_type} - {func})"): node.query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", exitcode = exitcode, message = 'Exception:') - with Scenario(f"Table - {data_type} - {func}"): + with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() table(name = table_name, data_type = data_type) diff --git a/tests/testflows/extended_precision_data_types/tests/comparison.py b/tests/testflows/extended_precision_data_types/tests/comparison.py index d3e5dcc0dbd..6f715e35b91 100644 --- a/tests/testflows/extended_precision_data_types/tests/comparison.py +++ b/tests/testflows/extended_precision_data_types/tests/comparison.py @@ -11,6 +11,7 @@ funcs = [ ] Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] +Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) @Examples('func int_type min max', Examples_list) @@ -51,7 +52,7 @@ def comp_int_table(self, func, int_type, min, max, node=None): """) @TestOutline(Scenario) -@Examples('func', funcs) +@Examples('func', Examples_list_dec) def comp_dec_inline(self, func, node=None): """Check comparison functions with Decimal256 using inline tests. """ @@ -67,7 +68,7 @@ def comp_dec_inline(self, func, node=None): """) @TestOutline(Scenario) -@Examples('func', funcs) +@Examples('func', Examples_list_dec) def comp_dec_table(self, func, node=None): """Check comparison functions with Decimal256 using table tests. """ diff --git a/tests/testflows/extended_precision_data_types/tests/conversion.py b/tests/testflows/extended_precision_data_types/tests/conversion.py index e46031ad29f..054288dc1c5 100644 --- a/tests/testflows/extended_precision_data_types/tests/conversion.py +++ b/tests/testflows/extended_precision_data_types/tests/conversion.py @@ -176,9 +176,23 @@ def to_decimal256(self, node=None): if node is None: node = self.context.node - for value in [1,min,max]: - output = node.query(f"SELECT toDecimal256(\'{value}\',0)").output - assert output == str(value), error() + with When(f"I check toDecimal256 with 0 scale with 1, {max}, and {min}"): + + for value in [1,min,max]: + output = node.query(f"SELECT toDecimal256(\'{value}\',0)").output + assert output == str(value), error() + + for scale in range(1,76): + + with When(f"I check toDecimal256 with {scale} scale with its max"): + output = node.query(f"SELECT toDecimal256(\'{10**(76-scale)-1}\',{scale})").output + assert float(output) == float(10**(76-scale)-1), error() + + with And(f"I check toDecimal256 with {scale} scale with its min"): + output = node.query(f"SELECT toDecimal256(\'{-10**(76-scale)+1}\',{scale})").output + assert float(output) == float(-10**(76-scale)+1), error() + +#, toDecimal256(\'{-10**(76-scale)+1}\',{scale}) @TestScenario @Requirements( diff --git a/tests/testflows/extended_precision_data_types/tests/logical.py b/tests/testflows/extended_precision_data_types/tests/logical.py index 36dbc97da97..18dc33f062e 100644 --- a/tests/testflows/extended_precision_data_types/tests/logical.py +++ b/tests/testflows/extended_precision_data_types/tests/logical.py @@ -9,6 +9,7 @@ funcs = [ ] Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] +Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) @Examples('func int_type min max', Examples_list) diff --git a/tests/testflows/extended_precision_data_types/tests/mathematical.py b/tests/testflows/extended_precision_data_types/tests/mathematical.py index 6062a00a7c9..b305c61ec06 100644 --- a/tests/testflows/extended_precision_data_types/tests/mathematical.py +++ b/tests/testflows/extended_precision_data_types/tests/mathematical.py @@ -37,8 +37,8 @@ funcs = [ ('hypot(1,', 1, 43), ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] +Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]}) - {data_type[0]}')]) for func in funcs for data_type in data_types] +Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]}) - Decimal256')]) for func in funcs] @TestOutline(Scenario) @Examples('func expected_result exitcode int_type min max', Examples_list) @@ -65,7 +65,7 @@ def math_int_inline(self, func, expected_result, exitcode, int_type, min, max, n with And(f"I check {func} with {int_type} using max and min"): execute_query(f""" - SELECT {func} to{int_type}(\'{max}\')), {func} to{int_type}(\'{min}\')) + SELECT round({func} to{int_type}(\'{max}\')), 7), round({func} to{int_type}(\'{min}\')), 7) """) @TestOutline(Scenario) @@ -94,7 +94,7 @@ def math_int_table(self, func, expected_result, exitcode, int_type, min, max, no for value in [1, max, min]: with And(f"I insert the output of {func} with {int_type} using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT to{int_type}OrZero( toString({func} to{int_type}(\'{value}\'))))") + node.query(f"INSERT INTO {table_name} SELECT round(to{int_type}OrZero( toString({func} to{int_type}(\'{value}\')))), 7)") with Then(f"I check the outputs of {func} with {int_type}"): execute_query(f""" @@ -129,7 +129,7 @@ def math_dec_inline(self, func, expected_result, exitcode, node=None): with And(f"I check {func} with Decimal256 using max and min"): execute_query(f""" - SELECT {func} toDecimal256(\'{max}\',0)), {func} toDecimal256(\'{min}\',0)) + SELECT round({func} toDecimal256(\'{max}\',0)),7), round({func} toDecimal256(\'{min}\',0)),7) """) @TestOutline(Scenario) @@ -161,7 +161,7 @@ def math_dec_table(self, func, expected_result, exitcode, node=None): for value in [1, max, min]: with When(f"I insert the output of {func} with Decimal256 using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT toDecimal256OrZero( toString({func} toDecimal256(\'{value}\',0))),0)") + node.query(f"INSERT INTO {table_name} SELECT round(toDecimal256OrZero( toString({func} toDecimal256(\'{value}\',0))),0), 7)") with Then(f"I check the outputs of {func} with Decimal256"): execute_query(f""" diff --git a/tests/testflows/extended_precision_data_types/tests/null.py b/tests/testflows/extended_precision_data_types/tests/null.py index 9ddaa94a245..f9b93f874bc 100644 --- a/tests/testflows/extended_precision_data_types/tests/null.py +++ b/tests/testflows/extended_precision_data_types/tests/null.py @@ -11,7 +11,8 @@ funcs = [ ('nullIf(1,', '\\N'), ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] +Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]}) - {data_type[0]}')]) for func in funcs for data_type in data_types] +Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]}) - Decimal256')]) for func in funcs] @TestOutline(Scenario) @Examples('func expected_result int_type min max', Examples_list) @@ -56,7 +57,7 @@ def null_int_table(self, func, expected_result, int_type, min, max, node=None): """) @TestOutline(Scenario) -@Examples('func expected_result', funcs) +@Examples('func expected_result', Examples_list_dec) def null_dec_inline(self, func, expected_result, node=None): """Check null function with Decimal256 using inline tests. """ @@ -76,7 +77,7 @@ def null_dec_inline(self, func, expected_result, node=None): """) @TestOutline(Scenario) -@Examples('func expected_result', funcs) +@Examples('func expected_result', Examples_list_dec) def null_dec_table(self, func, expected_result, node=None): """Check null function with Decimal256 using table tests. """ diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index c868d4a0d92..c7a264a9c27 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -30,7 +30,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) finally: join(tasks) From 1d64786e8168cb1a6ae24d4900f6a4543958aa24 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 15 Jun 2021 18:14:19 +0200 Subject: [PATCH 230/491] Removing commented code --- .../testflows/extended_precision_data_types/tests/conversion.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/tests/conversion.py b/tests/testflows/extended_precision_data_types/tests/conversion.py index 054288dc1c5..b98958009a0 100644 --- a/tests/testflows/extended_precision_data_types/tests/conversion.py +++ b/tests/testflows/extended_precision_data_types/tests/conversion.py @@ -192,8 +192,6 @@ def to_decimal256(self, node=None): output = node.query(f"SELECT toDecimal256(\'{-10**(76-scale)+1}\',{scale})").output assert float(output) == float(-10**(76-scale)+1), error() -#, toDecimal256(\'{-10**(76-scale)+1}\',{scale}) - @TestScenario @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_ToMySQL("1.0"), From 7b9c8edaf564f52bbef61ee239e1b8db12380ee0 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 15 Jun 2021 19:24:18 +0300 Subject: [PATCH 231/491] Fix Logical Error in min/maxMap --- .../AggregateFunctionSumMap.h | 86 ++++++++----------- .../0_stateless/01280_min_map_max_map.sql | 6 ++ 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index ec2f24d12cb..a4b6f3c967b 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -441,39 +441,32 @@ class FieldVisitorMax : public StaticVisitor { private: const Field & rhs; + + template + bool compareImpl(FieldType & x) const + { + auto val = get(rhs); + if (val > x) + { + x = val; + return true; + } + + return false; + } + public: explicit FieldVisitorMax(const Field & rhs_) : rhs(rhs_) {} bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Array &) const { throw Exception("Cannot compare Arrays", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Tuple &) const { throw Exception("Cannot compare Tuples", ErrorCodes::LOGICAL_ERROR); } bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot compare AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Array & x) const { return compareImpl(x); } + bool operator() (Tuple & x) const { return compareImpl(x); } template - bool operator() (DecimalField & x) const - { - auto val = get>(rhs); - if (val > x) - { - x = val; - return true; - } - - return false; - } - + bool operator() (DecimalField & x) const { return compareImpl>(x); } template - bool operator() (T & x) const - { - auto val = get(rhs); - if (val > x) - { - x = val; - return true; - } - - return false; - } + bool operator() (T & x) const { return compareImpl(x); } }; /** Implements `Min` operation. @@ -483,39 +476,32 @@ class FieldVisitorMin : public StaticVisitor { private: const Field & rhs; + + template + bool compareImpl(FieldType & x) const + { + auto val = get(rhs); + if (val < x) + { + x = val; + return true; + } + + return false; + } + public: explicit FieldVisitorMin(const Field & rhs_) : rhs(rhs_) {} bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } - bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); } bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } + bool operator() (Array & x) const { return compareImpl(x); } + bool operator() (Tuple & x) const { return compareImpl(x); } template - bool operator() (DecimalField & x) const - { - auto val = get>(rhs); - if (val < x) - { - x = val; - return true; - } - - return false; - } - + bool operator() (DecimalField & x) const { return compareImpl>(x); } template - bool operator() (T & x) const - { - auto val = get(rhs); - if (val < x) - { - x = val; - return true; - } - - return false; - } + bool operator() (T & x) const { return compareImpl(x); } }; diff --git a/tests/queries/0_stateless/01280_min_map_max_map.sql b/tests/queries/0_stateless/01280_min_map_max_map.sql index cc07998bb79..96fdfc61929 100644 --- a/tests/queries/0_stateless/01280_min_map_max_map.sql +++ b/tests/queries/0_stateless/01280_min_map_max_map.sql @@ -38,3 +38,9 @@ select maxMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([ select minMap(val, cnt) from values ('val Array(String), cnt Array(String)', (['A'], ['']), (['B'], [''])); select maxMap(val, cnt) from values ('val Array(String), cnt Array(String)', (['A'], ['']), (['B'], [''])); select sumMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([1], [0]), ([2], [0])); + +-- check working with arrays and tuples as values +select minMap([1, 1, 1], [[1, 2], [1], [1, 2, 3]]); +select maxMap([1, 1, 1], [[1, 2], [1], [1, 2, 3]]); +select minMap([1, 1, 1], [(1, 2), (1, 1), (1, 3)]); +select maxMap([1, 1, 1], [(1, 2), (1, 1), (1, 3)]); From f36c165fac1e317c6e26343332f40e9c138aa2fb Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 15 Jun 2021 19:29:20 +0300 Subject: [PATCH 232/491] Update test reference --- tests/queries/0_stateless/01280_min_map_max_map.reference | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01280_min_map_max_map.reference b/tests/queries/0_stateless/01280_min_map_max_map.reference index abb500553fd..37a8bfcc643 100644 --- a/tests/queries/0_stateless/01280_min_map_max_map.reference +++ b/tests/queries/0_stateless/01280_min_map_max_map.reference @@ -27,3 +27,7 @@ (['A','B'],['','']) (['A','B'],['','']) ([],[]) +([1],[[1]]) +([1],[[1,2,3]]) +([1],[(1,1)]) +([1],[(1,3)]) From b29804752717749cb3f1056c2f2611219d92e58c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 15 Jun 2021 19:29:20 +0300 Subject: [PATCH 233/491] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index f15a92860de..494917d0794 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -242,3 +242,4 @@ 01882_check_max_parts_to_merge_at_once 01892_setting_limit_offset_distributed 01901_test_attach_partition_from +01910_view_dictionary From 81693995898fd7e61fb9c4858b622821b69a52b8 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Tue, 15 Jun 2021 18:42:05 +0200 Subject: [PATCH 234/491] Rounding precision controlled by one variable --- tests/testflows/extended_precision_data_types/common.py | 2 ++ .../extended_precision_data_types/tests/arithmetic.py | 8 ++++---- .../extended_precision_data_types/tests/mathematical.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/common.py b/tests/testflows/extended_precision_data_types/common.py index 1c852bbf935..ebd0a6cac45 100644 --- a/tests/testflows/extended_precision_data_types/common.py +++ b/tests/testflows/extended_precision_data_types/common.py @@ -8,6 +8,8 @@ from testflows.asserts import values, error, snapshot from helpers.common import * +rounding_precision = 7 + @contextmanager def allow_experimental_bigint(node): """Enable experimental big int setting in Clickhouse. diff --git a/tests/testflows/extended_precision_data_types/tests/arithmetic.py b/tests/testflows/extended_precision_data_types/tests/arithmetic.py index db15e6fcfab..49d7ee1fcb3 100644 --- a/tests/testflows/extended_precision_data_types/tests/arithmetic.py +++ b/tests/testflows/extended_precision_data_types/tests/arithmetic.py @@ -63,7 +63,7 @@ def inline_check(self, arithmetic_func, expected_result, int_type, min, max, nod with When(f"I check {arithmetic_func} with {int_type} max and min value"): execute_query(f""" - SELECT round({arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), 7), round({arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1)), 7) + SELECT round({arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), {rounding_precision}), round({arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1)), {rounding_precision}) """) @TestOutline @@ -95,7 +95,7 @@ def table_check(self, arithmetic_func, expected_result, int_type, min, max, node else: with When(f"I insert {arithmetic_func} with {int_type} into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(1), to{int_type}(1)), 7)") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(1), to{int_type}(1)), {rounding_precision})") with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output @@ -125,7 +125,7 @@ def table_check(self, arithmetic_func, expected_result, int_type, min, max, node for value in [min, max]: with When(f"I insert {arithmetic_func} with {int_type} {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1)), 7)") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1)), {rounding_precision})") with Then(f"I check the table output of {arithmetic_func} with {int_type}"): execute_query(f""" @@ -191,7 +191,7 @@ def table_check_dec(self, arithmetic_func, expected_result, node=None): else: with When(f"I insert {arithmetic_func} with toDecimal256 into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0)), 7)") + node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0)), {rounding_precision})") with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output diff --git a/tests/testflows/extended_precision_data_types/tests/mathematical.py b/tests/testflows/extended_precision_data_types/tests/mathematical.py index b305c61ec06..65872b766dd 100644 --- a/tests/testflows/extended_precision_data_types/tests/mathematical.py +++ b/tests/testflows/extended_precision_data_types/tests/mathematical.py @@ -65,7 +65,7 @@ def math_int_inline(self, func, expected_result, exitcode, int_type, min, max, n with And(f"I check {func} with {int_type} using max and min"): execute_query(f""" - SELECT round({func} to{int_type}(\'{max}\')), 7), round({func} to{int_type}(\'{min}\')), 7) + SELECT round({func} to{int_type}(\'{max}\')), {rounding_precision}), round({func} to{int_type}(\'{min}\')), {rounding_precision}) """) @TestOutline(Scenario) @@ -94,7 +94,7 @@ def math_int_table(self, func, expected_result, exitcode, int_type, min, max, no for value in [1, max, min]: with And(f"I insert the output of {func} with {int_type} using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT round(to{int_type}OrZero( toString({func} to{int_type}(\'{value}\')))), 7)") + node.query(f"INSERT INTO {table_name} SELECT round(to{int_type}OrZero( toString({func} to{int_type}(\'{value}\')))), {rounding_precision})") with Then(f"I check the outputs of {func} with {int_type}"): execute_query(f""" @@ -129,7 +129,7 @@ def math_dec_inline(self, func, expected_result, exitcode, node=None): with And(f"I check {func} with Decimal256 using max and min"): execute_query(f""" - SELECT round({func} toDecimal256(\'{max}\',0)),7), round({func} toDecimal256(\'{min}\',0)),7) + SELECT round({func} toDecimal256(\'{max}\',0)),{rounding_precision}), round({func} toDecimal256(\'{min}\',0)),{rounding_precision}) """) @TestOutline(Scenario) From c1e57efda3b99fa7e2ef36985e4f8c9963ffaa52 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 15 Jun 2021 08:53:30 -0700 Subject: [PATCH 235/491] h3ToGeo - test h3ToGeo(geoToH3(lat, lon, res)) --- .../0_stateless/01906_h3_to_geo.reference | 16 +++++++++ tests/queries/0_stateless/01906_h3_to_geo.sql | 35 ++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01906_h3_to_geo.reference b/tests/queries/0_stateless/01906_h3_to_geo.reference index 5a45c5f3c30..93e8600576c 100644 --- a/tests/queries/0_stateless/01906_h3_to_geo.reference +++ b/tests/queries/0_stateless/01906_h3_to_geo.reference @@ -14,3 +14,19 @@ (15.642428355535966,40.285813505163574) (-76.53411447979884,54.5560449693637) (8.19906334981474,67.69370966550179) +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok diff --git a/tests/queries/0_stateless/01906_h3_to_geo.sql b/tests/queries/0_stateless/01906_h3_to_geo.sql index c78270d2808..aa6ecca1754 100644 --- a/tests/queries/0_stateless/01906_h3_to_geo.sql +++ b/tests/queries/0_stateless/01906_h3_to_geo.sql @@ -23,6 +23,39 @@ INSERT INTO h3_indexes VALUES (639763125756281263); INSERT INTO h3_indexes VALUES (644178757620501158); -select h3ToGeo(h3_index) from h3_indexes order by h3_index; +SELECT h3ToGeo(h3_index) FROM h3_indexes ORDER BY h3_index; DROP TABLE h3_indexes; + +DROP TABLE IF EXISTS h3_geo; + +-- compare if the results of h3ToGeo and geoToH3 are the same + +CREATE TABLE h3_geo(lat Float64, lon Float64, res UInt8) ENGINE = Memory; + +INSERT INTO h3_geo VALUES (-173.6412167681162, -14.130272474941535, 0); +INSERT INTO h3_geo VALUES (59.48137613600854, 58.020407687755686, 1); +INSERT INTO h3_geo VALUES (172.68095885060296, -83.6576608516349, 2); +INSERT INTO h3_geo VALUES (-94.46556851304558, -69.1999982492279, 3); +INSERT INTO h3_geo VALUES (-8.188263637093279, -55.856179102736284, 4); +INSERT INTO h3_geo VALUES (77.25594891852249, 47.39278564360122, 5); +INSERT INTO h3_geo VALUES (135.11348004704536, 36.60778126579667, 6); +INSERT INTO h3_geo VALUES (39.28534828967223, 49.07710003066973, 7); +INSERT INTO h3_geo VALUES (124.71163478198051, -27.481172161567258, 8); +INSERT INTO h3_geo VALUES (-147.4887686066785, 76.73237945824442, 9); +INSERT INTO h3_geo VALUES (86.63291906118863, -25.52526285188784, 10); +INSERT INTO h3_geo VALUES (23.27751790712118, 13.126101362212724, 11); +INSERT INTO h3_geo VALUES (-70.40163237204142, -63.12562536833242, 12); +INSERT INTO h3_geo VALUES (15.642428355535966, 40.285813505163574, 13); +INSERT INTO h3_geo VALUES (-76.53411447979884, 54.5560449693637, 14); +INSERT INTO h3_geo VALUES (8.19906334981474, 67.69370966550179, 15); + +SELECT result FROM ( + SELECT + (lat, lon) AS input_geo, + h3ToGeo(geoToH3(lat, lon, res)) AS output_geo, + if(input_geo = output_geo, 'ok', 'fail') AS result + FROM h3_geo +); + +DROP TABLE h3_geo; From 2dae69a40b35e577692a9ad08179d20915cef6e4 Mon Sep 17 00:00:00 2001 From: "christophe.kalenzaga" Date: Tue, 15 Jun 2021 18:58:20 +0200 Subject: [PATCH 236/491] add == operators in sequence*() aggregate fuctions --- .../aggregate-functions/parametric-functions.md | 2 +- .../aggregate-functions/parametric-functions.md | 2 +- .../aggregate-functions/parametric-functions.md | 2 +- .../aggregate-functions/parametric-functions.md | 2 +- .../AggregateFunctionSequenceMatch.h | 16 +++++++++++++++- .../00222_sequence_aggregate_function_family.sql | 4 ++++ 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 487074eca00..c77dfd6fa90 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -116,7 +116,7 @@ Type: `UInt8`. - `.*` — Matches any number of events. You do not need conditional arguments to match this element of the pattern. -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` matches events that occur more than 1800 seconds from each other. An arbitrary number of any events can lay between these events. You can use the `>=`, `>`, `<`, `<=` operators. +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` matches events that occur more than 1800 seconds from each other. An arbitrary number of any events can lay between these events. You can use the `>=`, `>`, `<`, `<=`, `==` operators. **Examples** diff --git a/docs/ja/sql-reference/aggregate-functions/parametric-functions.md b/docs/ja/sql-reference/aggregate-functions/parametric-functions.md index 1df84f2657c..9eba5baf8a7 100644 --- a/docs/ja/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ja/sql-reference/aggregate-functions/parametric-functions.md @@ -113,7 +113,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `.*` — Matches any number of events. You don't need conditional arguments to match this element of the pattern. -- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` お互いから1800秒以上発生するイベントと一致します。 これらのイベントの間に任意の数のイベントを配置できます。 を使用することができます `>=`, `>`, `<`, `<=` 演算子。 +- `(?t operator value)` — Sets the time in seconds that should separate two events. For example, pattern `(?1)(?t>1800)(?2)` お互いから1800秒以上発生するイベントと一致します。 これらのイベントの間に任意の数のイベントを配置できます。 を使用することができます `>=`, `>`, `<`, `<=`, `==` 演算子。 **例** diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index cf67dfb54ec..8942bfa3444 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -116,7 +116,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `.*` — соответствует любому количеству событий. Для этого элемента шаблона не надо задавать условия. -- `(?t operator value)` — устанавливает время в секундах, которое должно разделять два события. Например, шаблон `(?1)(?t>1800)(?2)` соответствует событиям, которые произошли более чем через 1800 секунд друг от друга. Между этими событиями может находиться произвольное количество любых событий. Операторы могут быть `>=`, `>`, `<`, `<=`. +- `(?t operator value)` — устанавливает время в секундах, которое должно разделять два события. Например, шаблон `(?1)(?t>1800)(?2)` соответствует событиям, которые произошли более чем через 1800 секунд друг от друга. Между этими событиями может находиться произвольное количество любых событий. Операторы могут быть `>=`, `>`, `<`, `<=`, `==`. **Примеры** diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index be9166e5737..fc0c7144305 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -112,7 +112,7 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) - `.*` — 匹配任何事件的数字。 不需要条件参数来匹配这个模式。 -- `(?t operator value)` — 分开两个事件的时间。 例如： `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=` 运算符。 +- `(?t operator value)` — 分开两个事件的时间。 例如： `(?1)(?t>1800)(?2)` 匹配彼此发生超过1800秒的事件。 这些事件之间可以存在任意数量的任何事件。 您可以使用 `>=`, `>`, `<`, `<=`, `==` 运算符。 **例** diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index f0fd1e33dca..a394c6d9aaf 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -188,7 +188,8 @@ private: TimeLessOrEqual, TimeLess, TimeGreaterOrEqual, - TimeGreater + TimeGreater, + TimeEqual }; struct PatternAction final @@ -250,6 +251,8 @@ private: type = PatternActionType::TimeGreaterOrEqual; else if (match(">")) type = PatternActionType::TimeGreater; + else if (match("==")) + type = PatternActionType::TimeEqual; else throw_exception("Unknown time condition"); @@ -474,6 +477,17 @@ protected: else if (++events_it == events_end && !do_backtrack()) break; } + else if (action_it->type == PatternActionType::TimeEqual) + { + if (events_it->first == base_it->first + action_it->extra) + { + back_stack.emplace(action_it, events_it, base_it); + base_it = events_it; + ++action_it; + } + else if (++events_it == events_end && !do_backtrack()) + break; + } else throw Exception{"Unknown PatternActionType", ErrorCodes::LOGICAL_ERROR}; diff --git a/tests/queries/0_stateless/00222_sequence_aggregate_function_family.sql b/tests/queries/0_stateless/00222_sequence_aggregate_function_family.sql index 1bec5be675a..3160bc8467b 100644 --- a/tests/queries/0_stateless/00222_sequence_aggregate_function_family.sql +++ b/tests/queries/0_stateless/00222_sequence_aggregate_function_family.sql @@ -25,6 +25,8 @@ select 0 = sequenceMatch('(?1)(?t<2)(?3)')(time, data = 0, data = 1, data = 2, d select 1 = sequenceMatch('(?2)(?t>=7)(?2)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; select 0 = sequenceMatch('(?2)(?t>7)(?2)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; select 1 = sequenceMatch('(?2)(?3)(?1)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; +select 0 = sequenceMatch('(?1)(?t==2)(?2)')(time, data = 1, data = 2) from sequence_test; +select 1 = sequenceMatch('(?1)(?t==1)(?2)')(time, data = 1, data = 2) from sequence_test; select count() = sequenceCount('')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; select count() = sequenceCount('.')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; @@ -47,5 +49,7 @@ select 0 = sequenceCount('(?1)(?t<2)(?3)')(time, data = 0, data = 1, data = 2, d select 1 = sequenceCount('(?2)(?t>=7)(?2)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; select 0 = sequenceCount('(?2)(?t>7)(?2)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; select 1 = sequenceCount('(?2)(?3)(?1)')(time, data = 0, data = 1, data = 2, data = 3) from sequence_test; +select 0 = sequenceCount('(?1)(?t==2)(?2)')(time, data = 1, data = 2) from sequence_test; +select 1 = sequenceCount('(?1)(?t==1)(?2)')(time, data = 1, data = 2) from sequence_test; drop table sequence_test; From 35f4a8985d39638a75f6d3b0a782d775308ee31c Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 15 Jun 2021 20:56:17 +0300 Subject: [PATCH 237/491] Translate to Russian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Перевел на русский язык. --- docs/ru/interfaces/formats.md | 56 +++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index f67997b58d6..a414efb2026 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1168,10 +1168,13 @@ SELECT * FROM topic1_stream; | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | | — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. -Неподдержанные типы данных Parquet: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +Неподдержанные типы данных Parquet: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. @@ -1197,6 +1200,53 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ `Arrow` — это Apache Arrow's "file mode" формат. Он предназначен для произвольного доступа в памяти. +### Соответствие типов данных {#sootvetstvie-tipov-dannykh-4} + +Таблица ниже содержит поддерживаемые типы данных и их соответствие [типам данных](../sql-reference/data-types/index.md) ClickHouse для запросов `INSERT` и `SELECT`. + +| Тип данных Arrow (`INSERT`) | Тип данных ClickHouse | Тип данных Arrow (`SELECT`) | +|-----------------------------|-----------------------------------------------------|-----------------------------| +| `UINT8`, `BOOL` | [UInt8](../sql-reference/data-types/int-uint.md) | `UINT8` | +| `INT8` | [Int8](../sql-reference/data-types/int-uint.md) | `INT8` | +| `UINT16` | [UInt16](../sql-reference/data-types/int-uint.md) | `UINT16` | +| `INT16` | [Int16](../sql-reference/data-types/int-uint.md) | `INT16` | +| `UINT32` | [UInt32](../sql-reference/data-types/int-uint.md) | `UINT32` | +| `INT32` | [Int32](../sql-reference/data-types/int-uint.md) | `INT32` | +| `UINT64` | [UInt64](../sql-reference/data-types/int-uint.md) | `UINT64` | +| `INT64` | [Int64](../sql-reference/data-types/int-uint.md) | `INT64` | +| `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT32` | +| `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `FLOAT64` | +| `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | +| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `UTF8` | +| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | +| `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. + +ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Arrow `DECIMAL` как `Decimal128`. + +Неподдержанные типы данных Arrow: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. + +Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Arrow. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. + +### Вставка данных {#vstavka-dannykh-3} + +Чтобы вставить в ClickHouse данные из файла в формате Arrow, используйте команду следующего вида: + +``` bash +$ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" +``` + +### Вывод данных {#vyvod-dannykh-3} + +Чтобы получить данные из таблицы ClickHouse и сохранить их в файл формата Arrow, используйте команду следующего вида: + +``` bash +$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow} +``` + ## ArrowStream {#data-format-arrow-stream} `ArrowStream` — это Apache Arrow's "stream mode" формат. Он предназначен для обработки потоков в памяти. @@ -1225,7 +1275,9 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | -| `-` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | + +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных ORC `DECIMAL` как `Decimal128`. From 03b92dc1b7ea1caaf4ed07da649c432dd1a94776 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 21:32:27 +0300 Subject: [PATCH 238/491] Support simple aggregate functions for SummingMergeTree. --- .../Algorithms/SummingSortedAlgorithm.cpp | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 19ead09ae4b..0258210df63 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -44,6 +45,11 @@ struct SummingSortedAlgorithm::AggregateDescription AlignedBuffer state; bool created = false; + /// For LowCardinality, convert is converted to nested type. nested_type is nullptr if no conversion needed. + /// This is used only for simple aggregate functions. + DataTypePtr nested_type; /// Nested type for LowCardinality, if it is. + DataTypePtr real_type; /// Type in header. + /// In case when column has type AggregateFunction: /// use the aggregate function from itself instead of 'function' above. bool is_agg_func_type = false; @@ -58,6 +64,8 @@ struct SummingSortedAlgorithm::AggregateDescription void init(AggregateFunctionPtr function_, bool is_simple_agg_func_type_ = false) { function = std::move(function_); + const auto & f = *function; + std::cerr << "agg func type " << typeid(f).name() << std::endl; add_function = function->getAddressOfAddFunction(); state.reset(function->sizeOfData(), function->alignOfData()); is_simple_agg_func_type = is_simple_agg_func_type_; @@ -266,6 +274,11 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( desc.init(simple->getFunction(), true); if (desc.function->allocatesMemoryInArena()) def.allocates_memory_in_arena = true; + + desc.real_type = column.type; + desc.nested_type = recursiveRemoveLowCardinality(desc.real_type); + if (desc.real_type->equals(*desc.nested_type)) + desc.nested_type = nullptr; } else if (!is_agg_func) { @@ -384,6 +397,12 @@ static MutableColumns getMergedDataColumns( columns.emplace_back(ColumnTuple::create(std::move(tuple_columns))); } + else if (desc.is_simple_agg_func_type) + { + const auto & type = desc.nested_type ? desc.nested_type + : desc.real_type; + columns.emplace_back(type->createColumn()); + } else columns.emplace_back(header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty()); } @@ -394,7 +413,7 @@ static MutableColumns getMergedDataColumns( return columns; } -static void preprocessChunk(Chunk & chunk) +static void preprocessChunk(Chunk & chunk, const SummingSortedAlgorithm::ColumnsDefinition & def) { auto num_rows = chunk.getNumRows(); auto columns = chunk.detachColumns(); @@ -402,6 +421,15 @@ static void preprocessChunk(Chunk & chunk) for (auto & column : columns) column = column->convertToFullColumnIfConst(); + for (const auto & desc : def.columns_to_aggregate) + { + if (desc.is_simple_agg_func_type && desc.nested_type) + { + auto & col = columns[desc.column_numbers[0]]; + col = recursiveRemoveLowCardinality(col); + } + } + chunk.setColumns(std::move(columns), num_rows); } @@ -427,6 +455,12 @@ static void postprocessChunk( for (size_t i = 0; i < tuple_size; ++i) res_columns[desc.column_numbers[i]] = assert_cast(*column).getColumnPtr(i); } + else if (desc.is_simple_agg_func_type && desc.nested_type) + { + const auto & from_type = desc.nested_type; + const auto & to_type = desc.real_type; + res_columns[desc.column_numbers[0]] = recursiveTypeConversion(std::move(column), from_type, to_type); + } else res_columns[desc.column_numbers[0]] = std::move(column); } @@ -620,6 +654,13 @@ void SummingSortedAlgorithm::SummingMergedData::addRowImpl(ColumnRawPtrs & raw_c if (desc.column_numbers.size() == 1) { auto & col = raw_columns[desc.column_numbers[0]]; + std::cerr << " ========= add_function " << reinterpret_cast(desc.add_function) + << " func " << reinterpret_cast(desc.function.get()) << " data " + << reinterpret_cast(desc.state.data()) + << " col " << reinterpret_cast(&col) + << " ar " << reinterpret_cast(arena.get()) << std::endl; + + std::cerr << "--- col " << col->dumpStructure() << " row " << row << std::endl; desc.add_function(desc.function.get(), desc.state.data(), &col, row, arena.get()); } else @@ -670,14 +711,14 @@ void SummingSortedAlgorithm::initialize(Inputs inputs) { for (auto & input : inputs) if (input.chunk) - preprocessChunk(input.chunk); + preprocessChunk(input.chunk, columns_definition); initializeQueue(std::move(inputs)); } void SummingSortedAlgorithm::consume(Input & input, size_t source_num) { - preprocessChunk(input.chunk); + preprocessChunk(input.chunk, columns_definition); updateCursor(input, source_num); } From c8cbb4445549640e37d6d877418460b1f0236c4b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 21:33:46 +0300 Subject: [PATCH 239/491] Support simple aggregate functions for SummingMergeTree. --- .../Merges/Algorithms/SummingSortedAlgorithm.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 0258210df63..8f5d0dbe43e 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -64,8 +64,6 @@ struct SummingSortedAlgorithm::AggregateDescription void init(AggregateFunctionPtr function_, bool is_simple_agg_func_type_ = false) { function = std::move(function_); - const auto & f = *function; - std::cerr << "agg func type " << typeid(f).name() << std::endl; add_function = function->getAddressOfAddFunction(); state.reset(function->sizeOfData(), function->alignOfData()); is_simple_agg_func_type = is_simple_agg_func_type_; @@ -654,13 +652,6 @@ void SummingSortedAlgorithm::SummingMergedData::addRowImpl(ColumnRawPtrs & raw_c if (desc.column_numbers.size() == 1) { auto & col = raw_columns[desc.column_numbers[0]]; - std::cerr << " ========= add_function " << reinterpret_cast(desc.add_function) - << " func " << reinterpret_cast(desc.function.get()) << " data " - << reinterpret_cast(desc.state.data()) - << " col " << reinterpret_cast(&col) - << " ar " << reinterpret_cast(arena.get()) << std::endl; - - std::cerr << "--- col " << col->dumpStructure() << " row " << row << std::endl; desc.add_function(desc.function.get(), desc.state.data(), &col, row, arena.get()); } else From 96d98ff020edcd202d149e851ebb7746465aa6d0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 15 Jun 2021 21:42:26 +0300 Subject: [PATCH 240/491] Add comment --- src/Storages/MergeTree/KeyCondition.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index efa238ccbfd..182602580ac 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -720,6 +720,9 @@ bool KeyCondition::canConstantBeWrappedByFunctions( /// We can assume that `modulo(...) = const` is the same as `moduloLegacy(...) = const`. /// Replace modulo to moduloLegacy in AST and check if we also have such a column. /// + /// We do not check this in canConstantBeWrappedByMonotonicFunctions. + /// The case `f(modulo(...))` for totally monotonic `f ` is consedered to be rare. + /// /// Note: for negative values, we can filter more partitions then needed. auto adjusted_ast = ast->clone(); KeyDescription::moduloToModuloLegacyRecursive(adjusted_ast); From f8e95abaf65ee2284cdbd58ae5ac9b8df539dab2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 15 Jun 2021 22:18:17 +0300 Subject: [PATCH 241/491] CMake dictionaries disable debug info fix --- src/CMakeLists.txt | 21 --------------------- src/Dictionaries/CMakeLists.txt | 12 ++++++++++++ 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4fa64e81f6c..88a6113b8fa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -257,27 +257,6 @@ if (USE_EMBEDDED_COMPILER) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () -if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") - # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. - set_source_files_properties( - Dictionaries/FlatDictionary.cpp - Dictionaries/HashedDictionary.cpp - Dictionaries/CacheDictionary.cpp - Dictionaries/IPAddressDictionary.cpp - Dictionaries/RangeHashedDictionary.cpp - Dictionaries/ComplexKeyHashedDictionary.cpp - Dictionaries/ComplexKeyCacheDictionary.cpp - Dictionaries/ComplexKeyCacheDictionary_generate1.cpp - Dictionaries/ComplexKeyCacheDictionary_generate2.cpp - Dictionaries/ComplexKeyCacheDictionary_generate3.cpp - Dictionaries/ODBCBlockInputStream.cpp - Dictionaries/HTTPDictionarySource.cpp - Dictionaries/LibraryDictionarySource.cpp - Dictionaries/ExecutableDictionarySource.cpp - Dictionaries/ClickHouseDictionarySource.cpp - PROPERTIES COMPILE_FLAGS -g0) -endif () - # Otherwise it will slow down stack traces printing too much. set_source_files_properties( Common/Elf.cpp diff --git a/src/Dictionaries/CMakeLists.txt b/src/Dictionaries/CMakeLists.txt index 540b9178f26..bc5f0dc9567 100644 --- a/src/Dictionaries/CMakeLists.txt +++ b/src/Dictionaries/CMakeLists.txt @@ -4,6 +4,18 @@ add_headers_and_sources(clickhouse_dictionaries .) add_headers_and_sources(clickhouse_dictionaries "${CMAKE_CURRENT_BINARY_DIR}/generated/") +if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") + + # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. + set_source_files_properties( + FlatDictionary.cpp + HashedDictionary.cpp + CacheDictionary.cpp + RangeHashedDictionary.cpp + DirectDictionary.cpp + PROPERTIES COMPILE_FLAGS -g0) +endif () + list(REMOVE_ITEM clickhouse_dictionaries_sources DictionaryFactory.cpp DictionarySourceFactory.cpp DictionaryStructure.cpp getDictionaryConfigurationFromAST.cpp) list(REMOVE_ITEM clickhouse_dictionaries_headers DictionaryFactory.h DictionarySourceFactory.h DictionaryStructure.h getDictionaryConfigurationFromAST.h) From 6e9589f6abc36d4c96860f87f385093ac532505b Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 15 Jun 2021 17:41:07 -0300 Subject: [PATCH 242/491] test for null_array_orc_load --- .../00900_null_array_orc_load.reference | 3 ++ .../0_stateless/00900_null_array_orc_load.sh | 27 ++++++++++++++++++ .../0_stateless/data_orc/test_null_array.orc | Bin 0 -> 442 bytes 3 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/00900_null_array_orc_load.reference create mode 100644 tests/queries/0_stateless/00900_null_array_orc_load.sh create mode 100644 tests/queries/0_stateless/data_orc/test_null_array.orc diff --git a/tests/queries/0_stateless/00900_null_array_orc_load.reference b/tests/queries/0_stateless/00900_null_array_orc_load.reference new file mode 100644 index 00000000000..b82db8ca1ef --- /dev/null +++ b/tests/queries/0_stateless/00900_null_array_orc_load.reference @@ -0,0 +1,3 @@ +[0] ['Test 0'] +[NULL] [NULL] +[] [] diff --git a/tests/queries/0_stateless/00900_null_array_orc_load.sh b/tests/queries/0_stateless/00900_null_array_orc_load.sh new file mode 100644 index 00000000000..a303e50a04a --- /dev/null +++ b/tests/queries/0_stateless/00900_null_array_orc_load.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DATA_FILE=$CUR_DIR/data_orc/test_null_array.orc + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_null_array_orc" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_null_array_orc(col0 Array(Nullable(Int64)),col1 Array(Nullable(String))) ENGINE = Memory" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into test_null_array_orc format ORC" +${CLICKHOUSE_CLIENT} --query="select * from test_null_array_orc" + +${CLICKHOUSE_CLIENT} --query="drop table test_null_array_orc" + +# +# test_null_array.orc is impossible to create using CH because it stores NULL instead of empty array +# but CH able to ingest it as empty array [] +# +# import pyorc +# with open("test_null_array.orc", "wb") as data: +# with pyorc.Writer(data, "struct,col1:array>") as writer: +# writer.write(([0], ["Test 0"])) +# writer.write(([None], [None])) +# writer.write((None, None)) +# +# diff --git a/tests/queries/0_stateless/data_orc/test_null_array.orc b/tests/queries/0_stateless/data_orc/test_null_array.orc new file mode 100644 index 0000000000000000000000000000000000000000..a70dcabd8a207480cd9fe6d13dc0196a1e60c678 GIT binary patch literal 442 zcmeYda+YOa;Nsz8VE_Ul77peB1{ns1$B&*Tr8h7rit#ODaAVK`3J5_2BpS6-dUeUWdfiFt zOL@!B9k`IQHLfOm^ORKk1kcs+4~5FPy)@i;(`GE1uljw`wJnFPTz$RnbG*;L*~{-d zu4GIBI&{;cq=yN72@I2b7<*!DWO|%(WO$f(gm@g}Tzs0E13f#sbyc?Ra5?C};pP2- z#ej+72~bml*pE{hM~@!VnV_7w^uhXrYg7&%R6cU#gwD}PbLJ;AsPT$Qm`LzQOxa<% zqeWuo0vSHOw3I}#9R}7dsRwe79y%yEz3yIbn77{EHDiRGg9Wz+i7f%up ykM_ta>}+Zjm@@4FXF|!02hR_3FmRL#FiJEuuqZJx@MxGCur+;QX7&$q76$-$NtMa~ literal 0 HcmV?d00001 From c873714698724001296c78f3db7b0b3dd6d1572b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 15 Jun 2021 23:52:29 +0300 Subject: [PATCH 243/491] improve replicated database tests --- docker/test/stateful/run.sh | 3 ++ docker/test/stateless/run.sh | 3 ++ src/Interpreters/executeDDLQueryOnCluster.cpp | 13 ++++--- tests/clickhouse-test | 36 ++++++++++++++----- tests/config/users.d/database_replicated.xml | 4 +-- ..._collapsing_merge_tree_zookeeper.reference | 6 ++-- ...sioned_collapsing_merge_tree_zookeeper.sql | 4 +-- 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 8d865431570..d8ea2153b36 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -112,12 +112,15 @@ timeout "$MAX_RUN_TIME" bash -c run_tests ||: ./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv +grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||: pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||: mv /var/log/clickhouse-server/stderr.log /test_output/ ||: if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||: fi if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then + grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||: + grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||: pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||: pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||: mv /var/log/clickhouse-server/stderr1.log /test_output/ ||: diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 6f7692f5e89..58b1d18a681 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -103,6 +103,7 @@ timeout "$MAX_RUN_TIME" bash -c run_tests ||: clickhouse-client -q "system flush logs" ||: +grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||: pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz & clickhouse-client -q "select * from system.query_log format TSVWithNamesAndTypes" | pigz > /test_output/query-log.tsv.gz & clickhouse-client -q "select * from system.query_thread_log format TSVWithNamesAndTypes" | pigz > /test_output/query-thread-log.tsv.gz & @@ -140,6 +141,8 @@ tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_l tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then + grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server1.log ||: + grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||: pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||: pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||: mv /var/log/clickhouse-server/stderr1.log /test_output/ ||: diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 99ece6bb14c..8a6abf7714f 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -175,17 +176,15 @@ BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & en if (context->getSettingsRef().distributed_ddl_task_timeout == 0) return io; - auto stream = std::make_shared(node_path, entry, context, hosts_to_wait); + BlockInputStreamPtr stream = std::make_shared(node_path, entry, context, hosts_to_wait); if (context->getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NONE) { /// Wait for query to finish, but ignore output - NullBlockOutputStream output{Block{}}; - copyData(*stream, output); - } - else - { - io.in = std::move(stream); + auto null_output = std::make_shared(stream->getHeader()); + stream = std::make_shared(std::move(stream), std::move(null_output)); } + + io.in = std::move(stream); return io; } diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f5346763e3a..9b6a61df730 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -36,11 +36,15 @@ MESSAGES_TO_RETRY = [ "Coordination::Exception: Session expired", "Coordination::Exception: Connection loss", "Coordination::Exception: Operation timeout", + "DB::Exception: Operation timeout", "Operation timed out", "ConnectionPoolWithFailover: Connection failed at try", - "DB::Exception: New table appeared in database being dropped or detached. Try again" + "DB::Exception: New table appeared in database being dropped or detached. Try again", + "is executing longer than distributed_ddl_task_timeout (=120)" # FIXME ] +MAX_RETRIES = 5 + class Terminated(KeyboardInterrupt): pass def signal_handler(sig, frame): @@ -258,7 +262,8 @@ def get_processlist(args): query = b"SHOW PROCESSLIST FORMAT Vertical" if args.replicated_database: query = b"SELECT materialize((hostName(), tcpPort())) as host, * " \ - b"FROM clusterAllReplicas('r', system.processes) WHERE query NOT LIKE '%system.processes%' FORMAT Vertical" + b"FROM clusterAllReplicas('test_cluster_database_replicated', system.processes) " \ + b"WHERE query NOT LIKE '%system.processes%' FORMAT Vertical" clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE) (stdout, _) = clickhouse_proc.communicate((query), timeout=20) return False, stdout.decode('utf-8') @@ -279,8 +284,16 @@ def get_stacktraces_from_gdb(server_pid): # collect server stacktraces from system.stack_trace table # it does not work in Sandbox -def get_stacktraces_from_clickhouse(client): +def get_stacktraces_from_clickhouse(client, replicated_database=False): try: + if replicated_database: + return subprocess.check_output("{} --allow_introspection_functions=1 --skip_unavailable_shards=1 --query " + "\"SELECT materialize((hostName(), tcpPort())) as host, thread_id, " + "arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " + "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " + "FROM clusterAllReplicas('test_cluster_database_replicated', 'system.stack_trace') " + "ORDER BY host, thread_id format Vertical\"".format(client), shell=True, stderr=subprocess.STDOUT).decode('utf-8') + return subprocess.check_output("{} --allow_introspection_functions=1 --query " "\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " @@ -289,7 +302,6 @@ def get_stacktraces_from_clickhouse(client): print("Error occured while receiving stack traces from client: {}".format(str(ex))) return None - def get_server_pid(server_tcp_port): # lsof does not work in stress tests for some reason cmd_lsof = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) @@ -317,7 +329,7 @@ SERVER_DIED = False exit_code = 0 stop_time = None queue = multiprocessing.Queue(maxsize=1) - +restarted_tests = [] # (test, stderr) # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total): def run_tests_array(all_tests_with_params): @@ -458,11 +470,12 @@ def run_tests_array(all_tests_with_params): else: counter = 1 while need_retry(stderr): + restarted_tests.append((case_file, stderr)) testcase_args = configure_testcase_args(args, case_file, suite_tmp_dir, stderr_file) proc, stdout, stderr, total_time = run_single_test(testcase_args, ext, server_logs_level, client_options, case_file, stdout_file, stderr_file) sleep(2**counter) counter += 1 - if counter > 6: + if MAX_RETRIES < counter: break if proc.returncode != 0: @@ -917,7 +930,7 @@ def main(args): clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') server_pid = get_server_pid(clickhouse_tcp_port) bt = None - if server_pid: + if server_pid and not args.replicated_database: print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) print("\nCollecting stacktraces from all running threads with gdb:") bt = get_stacktraces_from_gdb(server_pid) @@ -926,7 +939,7 @@ def main(args): bt = None if bt is None: print("\nCollecting stacktraces from system.stacktraces table:") - bt = get_stacktraces_from_clickhouse(args.client) + bt = get_stacktraces_from_clickhouse(args.client, args.replicated_database) if bt is None: print( colored( @@ -941,6 +954,13 @@ def main(args): else: print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) + if 0 < len(restarted_tests): + print("\nSome tests were restarted:\n") + for (test_case, stderr) in restarted_tests: + print(test_case) + print(stderr) + print("\n") + if total_tests_run == 0: print("No tests were run.") sys.exit(1) diff --git a/tests/config/users.d/database_replicated.xml b/tests/config/users.d/database_replicated.xml index cf59054c4d7..b19026e4a54 100644 --- a/tests/config/users.d/database_replicated.xml +++ b/tests/config/users.d/database_replicated.xml @@ -3,8 +3,8 @@ 1 none - 100 - 100 + 120 + 120 1 2 diff --git a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.reference b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.reference index 83a8162e5f4..d5ff3e12269 100644 --- a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.reference +++ b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.reference @@ -1,9 +1,9 @@ 1 1 1 -1 2 2 2 -1 -CREATE TABLE default.table_with_version_replicated_1\n(\n `key` UInt64,\n `value` String,\n `version` UInt8,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/t\', \'1\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_with_version_replicated_1\n(\n `key` UInt64,\n `value` String,\n `version` UInt8,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/{shard}/t\', \'1_{replica}\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 1 1 1 -1 2 2 2 -1 -CREATE TABLE default.table_with_version_replicated_1\n(\n `key` UInt64,\n `value` String,\n `version` UInt32,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/t\', \'1\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_with_version_replicated_1\n(\n `key` UInt64,\n `value` String,\n `version` UInt32,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/{shard}/t\', \'1_{replica}\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 1 1 2 1 2 2 2 -1 1 1 2 1 @@ -11,6 +11,6 @@ CREATE TABLE default.table_with_version_replicated_1\n(\n `key` UInt64,\n 3 3 65555 1 1 1 2 1 2 2 2 -1 -CREATE TABLE default.table_with_version_replicated_2\n(\n `key` UInt64,\n `value` String,\n `version` UInt32,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/t\', \'2\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_with_version_replicated_2\n(\n `key` UInt64,\n `value` String,\n `version` UInt32,\n `sign` Int8\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/default/test_01511/{shard}/t\', \'2_{replica}\', sign, version)\nORDER BY key\nSETTINGS index_granularity = 8192 1 1 2 1 2 2 2 -1 diff --git a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql index fe3476afbdb..17a6b8883a4 100644 --- a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql +++ b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql @@ -8,7 +8,7 @@ CREATE TABLE table_with_version_replicated_1 version UInt8, sign Int8 ) -ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/t', '1', sign, version) +ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/{shard}/t', '1_{replica}', sign, version) ORDER BY key; CREATE TABLE table_with_version_replicated_2 @@ -18,7 +18,7 @@ CREATE TABLE table_with_version_replicated_2 version UInt8, sign Int8 ) -ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/t', '2', sign, version) +ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/{shard}/t', '2_{replica}', sign, version) ORDER BY key; INSERT INTO table_with_version_replicated_1 VALUES (1, '1', 1, -1); From 60c8dc1c0b7613a1dae74af323395f210ccc3656 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 16 Jun 2021 00:48:38 +0300 Subject: [PATCH 244/491] Update ReplicatedMergeTreeRestartingThread.cpp --- .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 573d40951f7..1c9921aad1d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -356,7 +356,12 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.part_check_thread.stop(); /// Stop queue processing - storage.background_executor.finish(); + { + auto fetch_lock = storage.fetcher.blocker.cancel(); + auto merge_lock = storage.merger_mutator.merges_blocker.cancel(); + auto move_lock = storage.parts_mover.moves_blocker.cancel(); + storage.background_executor.finish(); + } LOG_TRACE(log, "Threads finished"); } From 1c506a023b2610c18a3da706b8c7622e6695f4af Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 15 Jun 2021 18:52:26 -0300 Subject: [PATCH 245/491] fix executable flag --- tests/queries/0_stateless/00900_null_array_orc_load.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/queries/0_stateless/00900_null_array_orc_load.sh diff --git a/tests/queries/0_stateless/00900_null_array_orc_load.sh b/tests/queries/0_stateless/00900_null_array_orc_load.sh old mode 100644 new mode 100755 From 8a709ce5ef7ca73aebe9a5fceefb0ee885087409 Mon Sep 17 00:00:00 2001 From: "christophe.kalenzaga" Date: Wed, 16 Jun 2021 00:13:57 +0200 Subject: [PATCH 246/491] add missing test reference file --- .../00222_sequence_aggregate_function_family.reference | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/00222_sequence_aggregate_function_family.reference b/tests/queries/0_stateless/00222_sequence_aggregate_function_family.reference index 010b7b79019..3966e9fa0e0 100644 --- a/tests/queries/0_stateless/00222_sequence_aggregate_function_family.reference +++ b/tests/queries/0_stateless/00222_sequence_aggregate_function_family.reference @@ -40,3 +40,7 @@ 1 1 1 +1 +1 +1 +1 From 67884ec50cc2112e09cdcf279efb662f6374dbca Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 16 Jun 2021 01:40:16 +0300 Subject: [PATCH 247/491] SimpleCache key constructor improvement --- base/common/SimpleCache.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/common/SimpleCache.h b/base/common/SimpleCache.h index 57247de696a..e412e921ab8 100644 --- a/base/common/SimpleCache.h +++ b/base/common/SimpleCache.h @@ -32,10 +32,11 @@ public: template Result operator() (Args &&... args) { + Key key{std::forward(args)...}; + { std::lock_guard lock(mutex); - Key key{std::forward(args)...}; auto it = cache.find(key); if (cache.end() != it) @@ -43,7 +44,7 @@ public: } /// The calculations themselves are not done under mutex. - Result res = f(std::forward(args)...); + Result res = std::apply(f, key); { std::lock_guard lock(mutex); @@ -57,11 +58,12 @@ public: template void update(Args &&... args) { - Result res = f(std::forward(args)...); + Key key{std::forward(args)...}; + + Result res = std::apply(f, key); + { std::lock_guard lock(mutex); - - Key key{std::forward(args)...}; cache[key] = std::move(res); } } From de9689379f0b160b8099f53757283d43a4116586 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 16 Jun 2021 06:56:55 +0300 Subject: [PATCH 248/491] Catch ErrnoException during parts cleaning Avoids noisy messages like: 2021.06.15 09:27:35.546982 [ 58176 ] {} auto DB::IBackgroundJobExecutor::jobExecutingTask()::(anonymous class)::operator()() const: Code: 481, e.displayText() = DB::ErrnoException: Cannot check modification time for file: /var/lib/clickhouse/store/c52/c52619f8-99bd-435e-8526-19f899bdb35e/tmp_insert_202106_7010_7010_0/, errno: 2, strerror: No such file or directory, Stack trace (when copying this message, always include the lines below): 0. DB::Exception::Exception() @ 0x903209a in /usr/lib/debug/.build-id/be/ea088a1830cb2f.debug 1. DB::throwFromErrnoWithPath() @ 0x9033089 in /usr/lib/debug/.build-id/be/ea088a1830cb2f.debug 2. FS::getModificationTime() @ 0x907b166 in /usr/lib/debug/.build-id/be/ea088a1830cb2f.debug 3. DB::DiskLocal::getLastModified() @ 0xf9a6669 in /usr/lib/debug/.build-id/be/ea088a1830cb2f.debug 4. DB::MergeTreeData::clearOldTemporaryDirectories(long) @ 0x1069a6de in /usr/lib/debug/.build-id/be/ea088a1830cb2f.debug --- src/Storages/MergeTree/MergeTreeData.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e76e91b5357..8ac6a37d4ed 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1161,6 +1161,16 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life disk->removeRecursive(it->path()); } } + /// see getModificationTime() + catch (const ErrnoException & e) + { + if (e.getErrno() == ENOENT) + { + /// If the file is already deleted, do nothing. + } + else + throw; + } catch (const fs::filesystem_error & e) { if (e.code() == std::errc::no_such_file_or_directory) From 784790610ebbce717ab016f581d9a1eed0fd1d94 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 08:52:23 +0300 Subject: [PATCH 249/491] Update tests (new results are correct) --- tests/queries/0_stateless/01035_avg.reference | 4 ++-- .../0_stateless/01721_dictionary_decimal_p_s.reference | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01035_avg.reference b/tests/queries/0_stateless/01035_avg.reference index a9f31de57e1..f8768d911d6 100644 --- a/tests/queries/0_stateless/01035_avg.reference +++ b/tests/queries/0_stateless/01035_avg.reference @@ -1,5 +1,5 @@ nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan --0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.493856 32355.57552 499999.5 499999.5 499999.5 499999.5 499999.5 499999.5 -0.000500002 0.49999949943727 -0.000005026740899901579 -0.000005257366687274546 +-0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.493856 32355.57552 499999.5 499999.5 499999.5 499999.5 499999.5 499999.5 -0.000500002 0.49999949943727 -0.000005 -0.000004999999999999992 -2767.546272 999999 --0.5000045261781699 +-0.50000449943727 diff --git a/tests/queries/0_stateless/01721_dictionary_decimal_p_s.reference b/tests/queries/0_stateless/01721_dictionary_decimal_p_s.reference index 066b4bd1d97..cfc5444a56e 100644 --- a/tests/queries/0_stateless/01721_dictionary_decimal_p_s.reference +++ b/tests/queries/0_stateless/01721_dictionary_decimal_p_s.reference @@ -1,9 +1,9 @@ -------- 42 -------- -42 14.0000 14.00000000 14.00000000 14.0000000000000000618637523926765281280 +42 14.0000 14.00000000 14.00000000 14.0000000000000000627860895963620057088 42 14.0000 14.00000000 14.00000000 14.0000 14.00000000 14.00000000 -------- 4999 -------- -4999 1666.3333 1666.33333333 1666.33333333 1633.3553612205046244471093725648757194800 +4999 1666.3333 1666.33333333 1666.33333333 1666.3333333333331934501138529985348370480 4999 1666.3333 1666.33333333 1666.33333333 1666.3333 1666.33333333 1666.33333333 -------- 5000 -------- From f0a532b2ad1a11c89ba5f87892c0116b1cf57d2c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 08:59:18 +0300 Subject: [PATCH 250/491] Fix flaky test 01520_client_print_query_id --- tests/queries/0_stateless/01520_client_print_query_id.expect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01520_client_print_query_id.expect b/tests/queries/0_stateless/01520_client_print_query_id.expect index c1d8b90ed39..b0ff5d9d165 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.expect +++ b/tests/queries/0_stateless/01520_client_print_query_id.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { From 13421b9cafa878aa707232cccd5c38eca0a4e816 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 09:01:57 +0300 Subject: [PATCH 251/491] Timeout of several seconds is not enough --- tests/queries/0_stateless/01176_mysql_client_interactive.expect | 2 +- tests/queries/0_stateless/01179_insert_values_semicolon.expect | 2 +- tests/queries/0_stateless/01180_client_syntax_errors.expect | 2 +- .../01293_client_interactive_vertical_multiline_long.expect | 2 +- .../01370_client_autocomplete_word_break_characters.expect | 2 +- .../01526_client_start_and_exit.expect-not-a-test-case | 2 +- .../0_stateless/01565_reconnect_after_client_error.expect | 2 +- .../01599_multiline_input_and_singleline_comments.sh | 2 +- tests/queries/0_stateless/01610_client_spawn_editor.sh | 2 +- .../0_stateless/01676_long_clickhouse_client_autocomplete.sh | 2 +- .../01755_client_highlight_multi_line_comment_regression.expect | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/01176_mysql_client_interactive.expect b/tests/queries/0_stateless/01176_mysql_client_interactive.expect index d592bbe1ce2..b2dc88a7795 100755 --- a/tests/queries/0_stateless/01176_mysql_client_interactive.expect +++ b/tests/queries/0_stateless/01176_mysql_client_interactive.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index 3814c71a062..0e65e5c4cbf 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { diff --git a/tests/queries/0_stateless/01180_client_syntax_errors.expect b/tests/queries/0_stateless/01180_client_syntax_errors.expect index 267e8edba10..c20982b2991 100755 --- a/tests/queries/0_stateless/01180_client_syntax_errors.expect +++ b/tests/queries/0_stateless/01180_client_syntax_errors.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline_long.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline_long.expect index 34d62307938..85eb97fb6f2 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline_long.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline_long.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect index a6d52b39918..e0d01d905bb 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 1 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case b/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case index 585c8c369dd..1b301612d6a 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 1 -set timeout 5 +set timeout 60 match_max 100000 spawn bash -c "$env(CLICKHOUSE_CLIENT_BINARY) $env(CLICKHOUSE_CLIENT_OPT)" diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index a977265d531..712fe4ff64a 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -4,7 +4,7 @@ # https://github.com/ClickHouse/ClickHouse/issues/19353 log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to fail diff --git a/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh b/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh index 9dc02a34592..248e8a06fb2 100755 --- a/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh +++ b/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} diff --git a/tests/queries/0_stateless/01610_client_spawn_editor.sh b/tests/queries/0_stateless/01610_client_spawn_editor.sh index 723fa761896..b372d82847c 100755 --- a/tests/queries/0_stateless/01610_client_spawn_editor.sh +++ b/tests/queries/0_stateless/01610_client_spawn_editor.sh @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} diff --git a/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh b/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh index 8e13f2dcaee..f68210c9c77 100755 --- a/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh +++ b/tests/queries/0_stateless/01676_long_clickhouse_client_autocomplete.sh @@ -17,7 +17,7 @@ function test_completion_word() # NOTE: here and below you should escape variables of the expect. timeout 60s expect << EOF log_user 0 -set timeout 3 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { diff --git a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect index 3e6f4fd9715..5543af4dd05 100755 --- a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect +++ b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 5 +set timeout 60 match_max 100000 # A default timeout action is to do nothing, change it to fail expect_after { From 46b230ddd7a6106f71302d438cf12d980377b6d3 Mon Sep 17 00:00:00 2001 From: Storozhuk Kostiantyn <56565543+sand6255@users.noreply.github.com> Date: Tue, 15 Jun 2021 22:33:58 +0300 Subject: [PATCH 252/491] Update InterpretersMySQLDDLQuery.cpp --- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index c0815ded0bc..fbd537781de 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -117,7 +117,7 @@ static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_defini return columns_name_and_type; } -static ColumnsDescription getColumnsDescriptionFromList(const NamesAndTypesList & columns_name_and_type, const ASTExpressionList * columns_definition) +static ColumnsDescription createColumnsDescription(const NamesAndTypesList & columns_name_and_type, const ASTExpressionList * columns_definition) { if (columns_name_and_type.size() != columns_definition->children.size()) throw Exception("Columns of different size provided.", ErrorCodes::LOGICAL_ERROR); @@ -425,7 +425,7 @@ ASTs InterpreterCreateImpl::getRewrittenQueries( NamesAndTypesList columns_name_and_type = getColumnsList(create_defines->columns); const auto & [primary_keys, unique_keys, keys, increment_columns] = getKeys(create_defines->columns, create_defines->indices, context, columns_name_and_type); - ColumnsDescription columns_description = getColumnsDescriptionFromList(columns_name_and_type, create_defines->columns); + ColumnsDescription columns_description = createColumnsDescription(columns_name_and_type, create_defines->columns); if (primary_keys.empty()) throw Exception("The " + backQuoteIfNeed(mysql_database) + "." + backQuoteIfNeed(create_query.table) From 8a7af3bc23a5d52892b0e711d76a76391adee744 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 16 Jun 2021 09:38:33 +0300 Subject: [PATCH 253/491] Update libpqxx --- contrib/libpqxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libpqxx b/contrib/libpqxx index ec49964f89e..357608d11b7 160000 --- a/contrib/libpqxx +++ b/contrib/libpqxx @@ -1 +1 @@ -Subproject commit ec49964f89eddc4e117fb93156f8ae6c75d33a1f +Subproject commit 357608d11b7a1961c3fb7db2ef9a5dbb2e87da77 From ea7b5497c0c4c0ce399af403a8aebcdedbbb50bb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 09:50:21 +0300 Subject: [PATCH 254/491] Fix behaviour of quantileDeterministic function --- .../ReservoirSamplerDeterministic.h | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 468a5c7c6a0..2687e0b4a67 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -59,9 +59,10 @@ template class ReservoirSamplerDeterministic { - bool good(const UInt32 hash) +private: + bool good(UInt32 hash) const { - return !(hash & skip_mask); + return (hash & skip_mask) == 0; } public: @@ -77,15 +78,12 @@ public: total_values = 0; } - void insert(const T & v, const UInt64 determinator) + void insert(const T & v, UInt64 determinator) { if (isNaN(v)) return; - const UInt32 hash = intHash64(determinator); - if (!good(hash)) - return; - + UInt32 hash = intHash64(determinator); insertImpl(v, hash); sorted = false; ++total_values; @@ -144,8 +142,7 @@ public: setSkipDegree(b.skip_degree); for (const auto & sample : b.samples) - if (good(sample.second)) - insertImpl(sample.first, sample.second); + insertImpl(sample.first, sample.second); total_values += b.total_values; } @@ -220,10 +217,19 @@ private: void insertImpl(const T & v, const UInt32 hash) { + if (!good(hash)) + return; + /// Make a room for plus one element. while (samples.size() >= max_sample_size) + { setSkipDegree(skip_degree + 1); + /// Still good? + if (!good(hash)) + return; + } + samples.emplace_back(v, hash); } From bd0b2fbcf0ce6c24367b992752d1d66db6e1b599 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 09:51:13 +0300 Subject: [PATCH 255/491] Add tests --- .../01913_quantile_deterministic.reference | 20 +++++++++++++++++++ .../01913_quantile_deterministic.sh | 18 +++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/queries/0_stateless/01913_quantile_deterministic.reference create mode 100755 tests/queries/0_stateless/01913_quantile_deterministic.sh diff --git a/tests/queries/0_stateless/01913_quantile_deterministic.reference b/tests/queries/0_stateless/01913_quantile_deterministic.reference new file mode 100644 index 00000000000..a39d3d91e8a --- /dev/null +++ b/tests/queries/0_stateless/01913_quantile_deterministic.reference @@ -0,0 +1,20 @@ +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 +492708 diff --git a/tests/queries/0_stateless/01913_quantile_deterministic.sh b/tests/queries/0_stateless/01913_quantile_deterministic.sh new file mode 100755 index 00000000000..5a2c7279678 --- /dev/null +++ b/tests/queries/0_stateless/01913_quantile_deterministic.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS d" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE d (oid UInt64) ENGINE = MergeTree ORDER BY oid" +${CLICKHOUSE_CLIENT} --min_insert_block_size_rows 0 --min_insert_block_size_bytes 0 --max_block_size 8192 --query "insert into d select * from numbers(1000000)" + +# In previous ClickHouse versions there was a mistake that makes quantileDeterministic functions not really deterministic (in edge cases). + +for _ in {1..20}; +do + ${CLICKHOUSE_CLIENT} --query "SELECT medianDeterministic(oid, oid) FROM d" +done + +${CLICKHOUSE_CLIENT} --query "DROP TABLE d" From 0343c442bc3fb42ec7ac97da4198f119339ad9d1 Mon Sep 17 00:00:00 2001 From: Kostiantyn Storozhuk Date: Wed, 16 Jun 2021 15:12:39 +0800 Subject: [PATCH 256/491] Trigger Build From d1bba10be0972adac7d6d6fc00b6ee54816eaa5e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 10:38:36 +0300 Subject: [PATCH 257/491] Fix UBSan report in quantileTiming --- src/AggregateFunctions/AggregateFunctionQuantile.h | 4 ++-- .../queries/0_stateless/01914_ubsan_quantile_timing.reference | 1 + tests/queries/0_stateless/01914_ubsan_quantile_timing.sql | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01914_ubsan_quantile_timing.reference create mode 100644 tests/queries/0_stateless/01914_ubsan_quantile_timing.sql diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index e7e60fe1def..90745c7d749 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -113,8 +113,8 @@ public: if constexpr (std::is_same_v>) { - /// QuantileTiming only supports integers. - if (isNaN(value) || value > std::numeric_limits::max() || value < std::numeric_limits::min()) + /// QuantileTiming only supports unsigned integers. Too large values are also meaningless. + if (isNaN(value) || value > std::numeric_limits::max() || value < 0) return; } diff --git a/tests/queries/0_stateless/01914_ubsan_quantile_timing.reference b/tests/queries/0_stateless/01914_ubsan_quantile_timing.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01914_ubsan_quantile_timing.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01914_ubsan_quantile_timing.sql b/tests/queries/0_stateless/01914_ubsan_quantile_timing.sql new file mode 100644 index 00000000000..422dd3bbbad --- /dev/null +++ b/tests/queries/0_stateless/01914_ubsan_quantile_timing.sql @@ -0,0 +1 @@ +SELECT quantileTiming(-0.)(number / 1.1754943508222875e-38) FROM numbers(257); From 28bc0ac3da9df4c16850f200a505ba3c094da73f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 16 Jun 2021 10:41:42 +0300 Subject: [PATCH 258/491] Update 01526_client_start_and_exit.expect-not-a-test-case --- .../01526_client_start_and_exit.expect-not-a-test-case | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case b/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case index 1b301612d6a..585c8c369dd 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect-not-a-test-case @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 1 -set timeout 60 +set timeout 5 match_max 100000 spawn bash -c "$env(CLICKHOUSE_CLIENT_BINARY) $env(CLICKHOUSE_CLIENT_OPT)" From 9f025cd405130c7958cf1333bdcba4b53134fb7e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 11:19:30 +0300 Subject: [PATCH 259/491] Update test. --- ...lter_version_versioned_collapsing_merge_tree_zookeeper.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql index fe3476afbdb..9b6b81112f6 100644 --- a/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql +++ b/tests/queries/0_stateless/01511_alter_version_versioned_collapsing_merge_tree_zookeeper.sql @@ -8,7 +8,7 @@ CREATE TABLE table_with_version_replicated_1 version UInt8, sign Int8 ) -ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/t', '1', sign, version) +ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/{database}/test_01511/t', '1', sign, version) ORDER BY key; CREATE TABLE table_with_version_replicated_2 @@ -18,7 +18,7 @@ CREATE TABLE table_with_version_replicated_2 version UInt8, sign Int8 ) -ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/' || currentDatabase() || '/test_01511/t', '2', sign, version) +ENGINE ReplicatedVersionedCollapsingMergeTree('/clickhouse/{database}/test_01511/t', '2', sign, version) ORDER BY key; INSERT INTO table_with_version_replicated_1 VALUES (1, '1', 1, -1); From 779f5eca9449e209fff928925fe5ad84a16f349b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 11:25:52 +0300 Subject: [PATCH 260/491] Relax max test time for performance test; fix comment --- docker/test/performance-comparison/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index b69a1e0d3f6..13b18cda326 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -489,7 +489,7 @@ if args.report == 'main': text = tableStart('Test Times') text += tableHeader(columns, attrs) - allowed_average_run_time = 1.6 # 30 seconds per test at 7 runs + allowed_average_run_time = 3.75 # 60 seconds per test at (7 + 1) * 2 runs for r in rows: anchor = f'{currentTableAnchor()}.{r[0]}' total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers From 661524e3d08dd2f0f23603209b702ffa09dfcfd9 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 16 Jun 2021 12:11:31 +0300 Subject: [PATCH 261/491] Update StoragePostgreSQL.cpp --- src/Storages/StoragePostgreSQL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index ff2e12d8be3..b1842c6b79f 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -151,7 +151,7 @@ public: } } - inserter->stream.write_values(row); // NOLINT + inserter->stream.write_values(row); } } @@ -273,7 +273,7 @@ private: StreamTo(pqxx::connection & connection, pqxx::table_path table_, Names columns_) : tx(connection) , columns(std::move(columns_)) - , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_), connection.quote_columns(columns))) // NOLINT + , stream(pqxx::stream_to::raw_table(tx, connection.quote_table(table_), connection.quote_columns(columns))) { } From 57b78b06ea54db05ddef6d4f5251e1b2fa29aca6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 16 Jun 2021 12:41:33 +0300 Subject: [PATCH 262/491] Update tests --- .../queries/0_stateless/00273_quantiles.reference | 14 +++++++------- .../01533_quantile_deterministic_assert.reference | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/00273_quantiles.reference b/tests/queries/0_stateless/00273_quantiles.reference index aefb0084648..ac9db31bae4 100644 --- a/tests/queries/0_stateless/00273_quantiles.reference +++ b/tests/queries/0_stateless/00273_quantiles.reference @@ -6,17 +6,17 @@ [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] [0,1,10,50,99.6,199.7,299.8,399.9,500,600.1,700.2,800.3,900.4,950,990,999,1000] [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] -1 333334 [699144.2,835663,967429.2] [699999,833333,966666] -2 266667 [426549.5,536255.5,638957.6] [426665,533332,639999] +1 333334 [699140.3,835642,967430.8] [699999,833333,966666] +2 266667 [426546,536239,638933.4] [426665,533332,639999] 3 114285 [296938,342324,388778] [297142,342856,388570] -4 63492 [228370.2,254019.5,279351.4] [228571,253968,279364] -5 40404 [185603.4,202009,218107] [185858,202020,218181] -6 27972 [156598.7,167866,179118.3] [156643,167832,179020] +4 63492 [228369.80000000002,254011,279351.6] [228571,253968,279364] +5 40404 [185602.3,202005,218108.5] [185858,202020,218181] +6 27972 [156598.6,167864,179118.40000000002] [156643,167832,179020] 7 20513 [135400.8,143550,151792.6] [135384,143589,151794] 8 15686 [119239.20000000001,125463,131772.40000000002] [119215,125490,131764] -9 12384 [106510.20000000001,111539,116415.7] [106501,111455,116408] +9 12384 [106509.79999999999,111538,116415.8] [106501,111455,116408] 10 10025 [96223.2,100346,104288.7] [96240,100250,104260] -11 8282 [87732.8,91036,94410.20000000001] [87784,91097,94409] +11 8282 [87732.70000000001,91035,94408.6] [87784,91097,94409] 12 6957 [80694.6,83477,86259.4] [80694,83477,86260] 13 5925 [74666.40000000001,77036,79405.6] [74666,77036,79406] 14 5109 [69475.8,71519,73562.2] [69475,71519,73563] diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference index 231c72269ca..1f6132cdec2 100644 --- a/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference @@ -1 +1 @@ -3998 +3997.5 From 7c3926ac57e555b45faf4ce5d35698ec25b78e9e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 12:55:48 +0300 Subject: [PATCH 263/491] Update test_ttl_replicated --- tests/integration/test_ttl_replicated/test.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 4d8d36e538f..de5e5984082 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -394,14 +394,26 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): # after restart table can be in readonly mode exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL") - node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") - exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") + exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + # After OPTIMIZE TABLE, it is not guaranteed that everything is merged. + # Possible scenario (for test_ttl_group_by): + # 1. Two independent merges assigned: [0_0, 1_1] -> 0_1 and [2_2, 3_3] -> 2_3 + # 2. Another one merge assigned: [0_1, 2_3] -> 0_3 + # 3. Merge to 0_3 is delayed: + # `Not executing log entry for part 0_3 because 2 merges with TTL already executing, maximum 2 + # 4. OPTIMIZE FINAL does nothing, cause there is an entry for 0_3 + # + # So, let's also sync replicas for node_right (for now). + exec_query_with_retry(node_right, "SYSTEM SYNC REPLICA test_ttl_delete") + node_right.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) + node_right.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + assert node_left.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" assert node_right.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" From f5aa447db7830763d46afb027b2f1c8aa2bd2300 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 16 Jun 2021 13:12:11 +0300 Subject: [PATCH 264/491] fix 00601_kill_running_query --- .../0_stateless/00601_kill_running_query.reference | 2 +- tests/queries/0_stateless/00601_kill_running_query.sh | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/00601_kill_running_query.reference b/tests/queries/0_stateless/00601_kill_running_query.reference index d00491fd7e5..e200d0ebbc2 100644 --- a/tests/queries/0_stateless/00601_kill_running_query.reference +++ b/tests/queries/0_stateless/00601_kill_running_query.reference @@ -1 +1 @@ -1 +waiting test_00601 default SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k) diff --git a/tests/queries/0_stateless/00601_kill_running_query.sh b/tests/queries/0_stateless/00601_kill_running_query.sh index e4e3ee98877..6cb511b7375 100755 --- a/tests/queries/0_stateless/00601_kill_running_query.sh +++ b/tests/queries/0_stateless/00601_kill_running_query.sh @@ -6,7 +6,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -o pipefail -${CLICKHOUSE_CURL_COMMAND} -q --max-time 30 -sS "$CLICKHOUSE_URL?query_id=hello" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(20000000) GROUP BY k)' | wc -l & -sleep 0.1 # First query (usually) should be received by the server after this sleep. -$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = 'hello' FORMAT Null" +function wait_for_query_to_start() +{ + while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done +} + +${CLICKHOUSE_CURL_COMMAND} -q --max-time 30 -sS "$CLICKHOUSE_URL&query_id=test_00601" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k)' > /dev/null & +wait_for_query_to_start 'test_00601' +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = 'test_00601'" wait From 5b7ca80da29f9dc0f5ab73fa17fc64e7935c9713 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 16 Jun 2021 13:19:18 +0300 Subject: [PATCH 265/491] Update clickhouse-test --- tests/clickhouse-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f5346763e3a..a3bb6913fa4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -522,13 +522,13 @@ def run_tests_array(all_tests_with_params): status += " - result differs with reference:\n{}\n".format(diff) status += 'Database: ' + testcase_args.testcase_database else: - if testcase_args.test_runs > 1 and total_time > 30 and 'long' not in name: + if testcase_args.test_runs > 1 and total_time > 60 and 'long' not in name: # We're in Flaky Check mode, check the run time as well while we're at it. failures += 1 failures_chain += 1 status += MSG_FAIL status += print_test_time(total_time) - status += " - Test runs too long (> 30s). Make it faster.\n" + status += " - Test runs too long (> 60s). Make it faster.\n" status += 'Database: ' + testcase_args.testcase_database else: passed_total += 1 From 92a028f8f0987e3d7ea20ca59dd877f097a3b405 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 16 Jun 2021 13:26:04 +0300 Subject: [PATCH 266/491] Update clickhouse-test --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 70cfc821546..fe9f010456b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -954,7 +954,7 @@ def main(args): else: print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) - if 0 < len(restarted_tests): + if len(restarted_tests) > 0: print("\nSome tests were restarted:\n") for (test_case, stderr) in restarted_tests: print(test_case) From f877c7ee51d951fea3d7baf6e2c553c778bf7f7e Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 16 Jun 2021 13:47:46 +0300 Subject: [PATCH 267/491] Apply suggestions from code review Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- .../external-dicts-dict-sources.md | 12 +++++++----- .../external-dicts-dict-sources.md | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 0077f618c04..9a24f5d5aa1 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -60,9 +60,9 @@ SETTINGS(format_csv_allow_single_quotes = 0) Types of sources (`source_type`): - [Local file](#dicts-external_dicts_dict_sources-local_file) -- [Executable file](#dicts-external_dicts_dict_sources-executable) -- [HTTP(s)](#dicts-external_dicts_dict_sources-http) +- [Executable File](#dicts-external_dicts_dict_sources-executable) - [Executable Pool](#dicts-external_dicts_dict_sources-executable_pool) +- [HTTP(s)](#dicts-external_dicts_dict_sources-http) - DBMS - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) @@ -122,14 +122,16 @@ Example of settings: Setting fields: - `command` — The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). -- `format` — The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. +- `format` — The file format. All the formats described in [Formats](../../../interfaces/formats.md#formats) are supported. - `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly — by the order of rows in the result. Default value is false. That dictionary source can be configured only via XML configuration. Creating dictionaries with executable source via DDL is disabled, otherwise, the DB user would be able to execute arbitrary binary on ClickHouse node. ## Executable Pool {#dicts-external_dicts_dict_sources-executable_pool} -Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary is stored using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. Executable pool will spawn pool of processes with specified command and keep them running until they exit. The program should read data from STDIN while it is available and output result to STDOUT, and it can wait for next block of data on stdin. ClickHouse will not close STDIN after processing a block of data but will pipe another chunk of data when needed. The executable script should be ready for this way of data processing — it should poll STDIN and flush data to STDOUT early. +Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary [is stored](external-dicts-dict-layout.md#ways-to-store-dictionaries-in-memory) using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. + +Executable pool will spawn pool of processes with specified command and keep them running until they exit. The program should read data from STDIN while it is available and output result to STDOUT, and it can wait for next block of data on STDIN. ClickHouse will not close STDIN after processing a block of data but will pipe another chunk of data when needed. The executable script should be ready for this way of data processing — it should poll STDIN and flush data to STDOUT early. Example of settings: @@ -150,7 +152,7 @@ Setting fields: - `command` — The absolute path to the executable file, or the file name (if the program directory is written to `PATH`). - `format` — The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. - `pool_size` — Size of pool. If 0 is specified as `pool_size` then there is no pool size restrictions. -- `command_termination_timeout` — Executable pool script, should contain main read-write loop. After dictionary is destroyed, pipe is closed, and executable file will have `command_termination_timeout` seconds to shutdown, before ClickHouse will send SIGTERM signal to child process. Specified in seconds. Default value is 10. Optional parameter. +- `command_termination_timeout` — Executable pool script should contain main read-write loop. After dictionary is destroyed, pipe is closed, and executable file will have `command_termination_timeout` seconds to shutdown, before ClickHouse will send SIGTERM signal to child process. Specified in seconds. Default value is 10. Optional parameter. - `max_command_execution_time` — Maximum executable script command execution time for processing block of data. Specified in seconds. Default value is 10. Optional parameter. - `implicit_key` — The executable source file can return only values, and the correspondence to the requested keys is determined implicitly — by the order of rows in the result. Default value is false. Optional parameter. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 67a8de29e2a..cc367f02484 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -61,8 +61,8 @@ SETTINGS(format_csv_allow_single_quotes = 0) - [Локальный файл](#dicts-external_dicts_dict_sources-local_file) - [Исполняемый файл](#dicts-external_dicts_dict_sources-executable) -- [HTTP(s)](#dicts-external_dicts_dict_sources-http) - [Исполняемый пул](#dicts-external_dicts_dict_sources-executable_pool) +- [HTTP(s)](#dicts-external_dicts_dict_sources-http) - СУБД: - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) @@ -122,14 +122,16 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) Поля настройки: - `command` — абсолютный путь к исполняемому файлу или имя файла (если каталог программы прописан в `PATH`). -- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». +- `format` — формат файла. Поддерживаются все форматы, описанные в разделе [Форматы](../../../interfaces/formats.md#formats). - `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. -Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse. +Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL запрещено. Иначе пользователь сможет выполнить произвольный бинарный файл на сервере ClickHouse. ## Исполняемый пул {#dicts-external_dicts_dict_sources-executable_pool} -Исполняемый пул позволяет загружать данные из пула процессов. Этот источник не работает с макетами словарей, которые требуют загрузки всех данных источника. Исполняемый пул работает, если тип размещения словаря `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct`. Исполняемый пул генерирует пул процессов с указанной командой и оставляет их активными, пока те не выйдут. Программа считывает данные из потока STDIN, пока он доступен, и выводит результат в поток STDOUT, кроме того возможно ожидание следующего блока данных из STDIN. ClickHouse не закрывает STDIN после обработки блока данных, но посылает следующую часть данных, когда это требуется. Исполняемый скрипт должен быть готов к такому способу передачи данных — он должен опросить STDIN и сбросить данные в STDOUT заранее. +Исполняемый пул позволяет загружать данные из пула процессов. Этот источник не работает со словарями, которые требуют загрузки всех данных из источника. Исполняемый пул работает словарями, которые размещаются [следующими способами](external-dicts-dict-layout.md#ways-to-store-dictionaries-in-memory): `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct`. + +Исполняемый пул генерирует пул процессов с помощью указанной команды и оставляет их активными, пока они не завершатся. Программа считывает данные из потока STDIN пока он доступен и выводит результат в поток STDOUT, а затем ожидает следующего блока данных из STDIN. ClickHouse не закрывает поток STDIN после обработки блока данных и отправляет в него следующую порцию данных, когда это требуется. Исполняемый скрипт должен быть готов к такому способу обработки данных — он должен заранее опрашивать STDIN и отправлять данные в STDOUT. Пример настройки: @@ -150,7 +152,7 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) - `command` — абсолютный путь к файлу или имя файла (если каталог программы записан в `PATH`). - `format` — формат файла. Поддерживаются все форматы, описанные в “[Форматы](../../../interfaces/formats.md#formats)”. - `pool_size` — размер пула. Если в поле `pool_size` указан 0, то размер пула не ограничен. -- `command_termination_timeout` — скрипт исполняемого пула, должен включать основной цикл чтения-записи. После уничтожения словаря канал закрывается. При этом исполняемый файл имеет `command_termination_timeout` секунд для завершения работы, прежде чем ClickHouse пошлет сигнал SIGTERM дочернему процессу. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. +- `command_termination_timeout` — скрипт исполняемого пула должен включать основной цикл чтения-записи. После уничтожения словаря канал закрывается. При этом исполняемый файл имеет `command_termination_timeout` секунд для завершения работы, прежде чем ClickHouse пошлет сигнал SIGTERM дочернему процессу. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. - `max_command_execution_time` — максимальное количество времени для исполняемого скрипта на обработку блока данных. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. - `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. @@ -762,4 +764,3 @@ Setting fields: - `where` – Условие выборки. Синтаксис для условий такой же как для `WHERE` выражения в PostgreSQL, для примера, `id > 10 AND id < 20`. Необязательный параметр. - `invalidate_query` – Запрос для проверки условия загрузки словаря. Необязательный параметр. Читайте больше в разделе [Обновление словарей](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). - From 864b35533e8869ba65dd084085985994ea9919f1 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 16 Jun 2021 13:53:09 +0300 Subject: [PATCH 268/491] small fixes --- .../external-dictionaries/external-dicts-dict-sources.md | 2 +- .../external-dictionaries/external-dicts-dict-sources.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 9a24f5d5aa1..84b4e8a0974 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -95,7 +95,7 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) Setting fields: - `path` – The absolute path to the file. -- `format` – The file format. All the formats described in “[Formats](../../../interfaces/formats.md#formats)” are supported. +- `format` – The file format. All the formats described in [Formats](../../../interfaces/formats.md#formats) are supported. When dictionary with source `FILE` is created via DDL command (`CREATE DICTIONARY ...`), the source file needs to be located in `user_files` directory, to prevent DB users accessing arbitrary file on ClickHouse node. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index cc367f02484..038724b3199 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -95,7 +95,7 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) Поля настройки: - `path` — абсолютный путь к файлу. -- `format` — формат файла. Поддерживаются все форматы, описанные в разделе «[Форматы](../../../interfaces/formats.md#formats)». +- `format` — формат файла. Поддерживаются все форматы, описанные в разделе [Форматы](../../../interfaces/formats.md#formats). Если словарь с источником `FILE` создается с помощью DDL-команды (`CREATE DICTIONARY ...`), источник словаря должен быть расположен в каталоге `user_files`. Иначе пользователи базы данных будут иметь доступ к произвольному файлу на узле ClickHouse. @@ -156,7 +156,7 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) - `max_command_execution_time` — максимальное количество времени для исполняемого скрипта на обработку блока данных. Указывается в секундах. Значение по умолчанию: 10. Необязательный параметр. - `implicit_key` — исходный исполняемый файл может возвращать только значения, а соответствие запрошенным ключам определено неявно — порядком строк в результате. Значение по умолчанию: false. -Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL отключено. Иначе пользователь базы данных сможет выполнить произвольный бинарный файл на узле ClickHouse. +Этот источник словаря может быть настроен только с помощью XML-конфигурации. Создание словарей с исполняемым источником с помощью DDL запрещено. Иначе пользователь сможет выполнить произвольный бинарный файл на сервере ClickHouse. ## HTTP(s) {#dicts-external_dicts_dict_sources-http} From b319c70e8e60ef8da7e2416424056beab28311fb Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 16 Jun 2021 13:39:42 +0300 Subject: [PATCH 269/491] Update 01049_join_low_card_bug.sql --- .../01049_join_low_card_bug.reference | 230 ++++++++++++++---- .../0_stateless/01049_join_low_card_bug.sql | 14 +- 2 files changed, 187 insertions(+), 57 deletions(-) diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.reference b/tests/queries/0_stateless/01049_join_low_card_bug.reference index b4ed8176652..74ea5fb8c27 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.reference +++ b/tests/queries/0_stateless/01049_join_low_card_bug.reference @@ -1,80 +1,202 @@ - LowCardinality(String) str LowCardinality(String) LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) - str LowCardinality(String) LowCardinality(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str str LowCardinality(String) LowCardinality(String) + str_r LowCardinality(String) LowCardinality(String) +str str LowCardinality(String) LowCardinality(String) + str_r LowCardinality(String) LowCardinality(String) +str str LowCardinality(String) LowCardinality(String) +str_l LowCardinality(String) LowCardinality(String) + str_r LowCardinality(String) LowCardinality(String) +str str LowCardinality(String) LowCardinality(String) +str_l LowCardinality(String) LowCardinality(String) + str_r LowCardinality(String) LowCardinality(String) str LowCardinality(String) LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) String - str LowCardinality(String) String - str LowCardinality(String) String - str LowCardinality(String) String - String +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) + LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str str LowCardinality(String) String + str_r LowCardinality(String) String +str str LowCardinality(String) String + str_r LowCardinality(String) String +str str LowCardinality(String) String +str_l LowCardinality(String) String + str_r LowCardinality(String) String +str str LowCardinality(String) String +str_l LowCardinality(String) String + str_r LowCardinality(String) String str String String str String - str String LowCardinality(String) - str String LowCardinality(String) - str String LowCardinality(String) - str String LowCardinality(String) - LowCardinality(String) +str_r String +str String +str_l String + String +str String +str_l String +str_r String +str str String LowCardinality(String) + str_r String LowCardinality(String) +str str String LowCardinality(String) + str_r String LowCardinality(String) +str str String LowCardinality(String) +str_l String LowCardinality(String) + str_r String LowCardinality(String) +str str String LowCardinality(String) +str_l String LowCardinality(String) + str_r String LowCardinality(String) str LowCardinality(String) LowCardinality(String) str LowCardinality(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) - str LowCardinality(String) Nullable(String) -\N Nullable(String) +str_r LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) + LowCardinality(String) +str LowCardinality(String) +str_l LowCardinality(String) +str_r LowCardinality(String) +str str LowCardinality(String) Nullable(String) + str_r LowCardinality(String) Nullable(String) +str str LowCardinality(String) Nullable(String) + str_r LowCardinality(String) Nullable(String) +str str LowCardinality(String) Nullable(String) +str_l \N LowCardinality(String) Nullable(String) + str_r LowCardinality(String) Nullable(String) +str str LowCardinality(String) Nullable(String) +str_l \N LowCardinality(String) Nullable(String) + str_r LowCardinality(String) Nullable(String) str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N str Nullable(String) LowCardinality(String) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N LowCardinality(Nullable(String)) -str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) +str_r Nullable(String) +str Nullable(String) +str_l Nullable(String) \N Nullable(String) str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +str str Nullable(String) LowCardinality(String) +\N str_r Nullable(String) LowCardinality(String) +str str Nullable(String) LowCardinality(String) +\N str_r Nullable(String) LowCardinality(String) +str str Nullable(String) LowCardinality(String) +str_l Nullable(String) LowCardinality(String) +\N str_r Nullable(String) LowCardinality(String) +str str Nullable(String) LowCardinality(String) +str_l Nullable(String) LowCardinality(String) +\N str_r Nullable(String) LowCardinality(String) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N LowCardinality(Nullable(String)) +str_r Nullable(String) +str Nullable(String) +str_l Nullable(String) +\N Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +str str Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +str_l \N Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +str_l \N Nullable(String) +\N str_r Nullable(String) str LowCardinality(Nullable(String)) \N LowCardinality(Nullable(String)) str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N str LowCardinality(Nullable(String)) -\N Nullable(String) +str_r LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +\N LowCardinality(Nullable(String)) +str LowCardinality(Nullable(String)) +str_l LowCardinality(Nullable(String)) +str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) +str str LowCardinality(Nullable(String)) +str_l \N LowCardinality(Nullable(String)) +\N str_r LowCardinality(Nullable(String)) str Nullable(String) \N Nullable(String) str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) -\N str Nullable(String) +str_r Nullable(String) +str Nullable(String) +str_l Nullable(String) +\N Nullable(String) +str Nullable(String) +str_l Nullable(String) +str_r Nullable(String) +str str Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +str_l \N Nullable(String) +\N str_r Nullable(String) +str str Nullable(String) +str_l \N Nullable(String) +\N str_r Nullable(String) + LowCardinality(String) +str LowCardinality(String) diff --git a/tests/queries/0_stateless/01049_join_low_card_bug.sql b/tests/queries/0_stateless/01049_join_low_card_bug.sql index 07558770abf..ae2fd9d3a43 100644 --- a/tests/queries/0_stateless/01049_join_low_card_bug.sql +++ b/tests/queries/0_stateless/01049_join_low_card_bug.sql @@ -12,9 +12,13 @@ CREATE TABLE nr (x Nullable(UInt32), lc Nullable(String)) ENGINE = Memory; CREATE TABLE l_lc (x UInt32, lc LowCardinality(String)) ENGINE = Memory; CREATE TABLE r_lc (x UInt32, lc LowCardinality(String)) ENGINE = Memory; -INSERT INTO r VALUES (0, 'str'); -INSERT INTO nr VALUES (0, 'str'); -INSERT INTO r_lc VALUES (0, 'str'); +INSERT INTO r VALUES (0, 'str'), (1, 'str_r'); +INSERT INTO nr VALUES (0, 'str'), (1, 'str_r'); +INSERT INTO r_lc VALUES (0, 'str'), (1, 'str_r'); + +INSERT INTO l VALUES (0, 'str'), (2, 'str_l'); +INSERT INTO nl VALUES (0, 'str'), (2, 'str_l'); +INSERT INTO l_lc VALUES (0, 'str'), (2, 'str_l'); -- @@ -136,6 +140,10 @@ SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l RIGHT JOIN r_lc AS r USING (lc) SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (x); SELECT l.lc, r.lc, toTypeName(l.lc) FROM nl AS l FULL JOIN r_lc AS r USING (lc); +SET join_use_nulls = 0; + +SELECT lc, toTypeName(lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY l.lc; + DROP TABLE l; DROP TABLE r; DROP TABLE nl; From ca66f655c8a0634266c16fb2335d2dab84630623 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 14:15:08 +0300 Subject: [PATCH 270/491] Use signal 15 --- tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 3b3dba32be3..70a52e42048 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1920,7 +1920,7 @@ class ClickHouseInstance: return None return None - def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=60): + def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=15): if not self.stay_alive: raise Exception("Cannot restart not stay alive container") self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root') From 81022aadd4006845418886cac3120b252fc3e054 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 14:54:34 +0300 Subject: [PATCH 271/491] increase timeout in 01532_execute_merges_on_single_replica --- .../0_stateless/01532_execute_merges_on_single_replica.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01532_execute_merges_on_single_replica.sql b/tests/queries/0_stateless/01532_execute_merges_on_single_replica.sql index 2fa89ed3464..69369321d26 100644 --- a/tests/queries/0_stateless/01532_execute_merges_on_single_replica.sql +++ b/tests/queries/0_stateless/01532_execute_merges_on_single_replica.sql @@ -115,7 +115,7 @@ SELECT arraySort(groupArrayIf(table, event_type = 'MergeParts')) AS mergers, arraySort(groupArrayIf(table, event_type = 'DownloadPart')) AS fetchers FROM system.part_log -WHERE (event_time > (now() - 40)) +WHERE (event_time > (now() - 120)) AND (table LIKE 'execute\\_on\\_single\\_replica\\_r%') AND (part_name NOT LIKE '%\\_0') AND (database = currentDatabase()) From 192fe1fc5bc6903e6657c553458de4b688a57c29 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 15:05:31 +0300 Subject: [PATCH 272/491] Add test. --- .../Algorithms/SummingSortedAlgorithm.cpp | 6 +++--- ..._and_simple_agg_function_with_lc.reference | 2 ++ ...ing_mt_and_simple_agg_function_with_lc.sql | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.reference create mode 100644 tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 8f5d0dbe43e..a31d4941535 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -45,8 +45,8 @@ struct SummingSortedAlgorithm::AggregateDescription AlignedBuffer state; bool created = false; - /// For LowCardinality, convert is converted to nested type. nested_type is nullptr if no conversion needed. - /// This is used only for simple aggregate functions. + /// Those types are used only for simple aggregate functions. + /// For LowCardinality, convert to nested type. nested_type is nullptr if no conversion needed. DataTypePtr nested_type; /// Nested type for LowCardinality, if it is. DataTypePtr real_type; /// Type in header. @@ -275,7 +275,7 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( desc.real_type = column.type; desc.nested_type = recursiveRemoveLowCardinality(desc.real_type); - if (desc.real_type->equals(*desc.nested_type)) + if (desc.real_type.get() == desc.nested_type.get()) desc.nested_type = nullptr; } else if (!is_agg_func) diff --git a/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.reference b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.reference new file mode 100644 index 00000000000..b9a964c80d0 --- /dev/null +++ b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.reference @@ -0,0 +1,2 @@ +1 x +1 y diff --git a/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql new file mode 100644 index 00000000000..b4d3bad6cae --- /dev/null +++ b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql @@ -0,0 +1,21 @@ +drop table if exists smta; + +CREATE TABLE smta +( + `k` Int64, + `a` AggregateFunction(max, Int64), + `city` SimpleAggregateFunction(max, LowCardinality(String)) +) +ENGINE = SummingMergeTree +ORDER BY k; + +insert into smta(k, city) values (1, 'x'); + +select k, city from smta; + +insert into smta(k, city) values (1, 'y'); +optimize table smta; + +select k, city from smta; + +drop table if exists smta; \ No newline at end of file From 39398b5589fe22da9da908a438f066cab45916b0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 15:06:14 +0300 Subject: [PATCH 273/491] Add test. --- .../01913_summing_mt_and_simple_agg_function_with_lc.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql index b4d3bad6cae..f4785c96312 100644 --- a/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql +++ b/tests/queries/0_stateless/01913_summing_mt_and_simple_agg_function_with_lc.sql @@ -18,4 +18,4 @@ optimize table smta; select k, city from smta; -drop table if exists smta; \ No newline at end of file +drop table if exists smta; From 0414d1b39ea7b5832df2b6b3401cee54cfaf6b1a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 16 Jun 2021 15:18:55 +0300 Subject: [PATCH 274/491] Setting min_count_to_compile_expression fix --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e9379186c0e..2aed174c088 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -104,7 +104,7 @@ class IColumn; \ M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \ M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \ - M(UInt64, min_count_to_compile_expression, 0, "The number of identical expressions before they are JIT-compiled", 0) \ + M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ From f4d587a17a7b189b798f1b0b0858b34a667d8f49 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:31:19 +0300 Subject: [PATCH 275/491] hardening --- tests/integration/helpers/cluster.py | 15 ++++++------- .../test_cluster_copier/test_three_nodes.py | 22 +++++++++---------- .../test.py | 2 +- .../integration/test_distributed_ddl/test.py | 7 +++--- .../integration/test_zookeeper_config/test.py | 2 ++ .../test_zookeeper_config/test_password.py | 2 ++ 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index cc5c428ed2e..d40505bcb89 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1421,9 +1421,9 @@ class ClickHouseCluster: instance.docker_client = self.docker_client instance.ip_address = self.get_instance_ip(instance.name) - logging.debug("Waiting for ClickHouse start...") + logging.debug("Waiting for ClickHouse start in {instance}, ip: {instance.ip_address}...") instance.wait_for_start(start_timeout) - logging.debug("ClickHouse started") + logging.debug("ClickHouse {instance} started") instance.client = Client(instance.ip_address, command=self.client_bin_path) @@ -1911,6 +1911,7 @@ class ClickHouseInstance: self.get_docker_handle().start() def wait_for_start(self, start_timeout=None, connection_timeout=None): + handle = self.get_docker_handle() if start_timeout is None or start_timeout <= 0: raise Exception("Invalid timeout: {}".format(start_timeout)) @@ -1933,11 +1934,10 @@ class ClickHouseInstance: return False while True: - handle = self.get_docker_handle() + handle.reload() status = handle.status if status == 'exited': - raise Exception("Instance `{}' failed to start. Container status: {}, logs: {}" - .format(self.name, status, handle.logs().decode('utf-8'))) + raise Exception(f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8')}") deadline = start_time + start_timeout # It is possible that server starts slowly. @@ -1947,9 +1947,8 @@ class ClickHouseInstance: current_time = time.time() if current_time >= deadline: - raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. " - "Container status: {}, logs: {}".format(self.name, self.ip_address, status, - handle.logs().decode('utf-8'))) + raise Exception(f"Timed out while waiting for instance `{self.name}' with ip address {self.ip_address} to start. " \ + f"Container status: {status}, logs: {handle.logs().decode('utf-8')}") socket_timeout = min(start_timeout, deadline - current_time) diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index e9fcad7247f..63b0bcc6679 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -48,11 +48,11 @@ class Task: node.query("DROP DATABASE IF EXISTS dailyhistory SYNC;") node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") - instance = cluster.instances['first'] + first = cluster.instances['first'] # daily partition database - instance.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") - instance.query("""CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events + first.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") + first.query("""CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -84,12 +84,12 @@ class Task: ORDER BY (tpep_pickup_datetime, id) PARTITION BY (toYYYYMMDD(tpep_pickup_datetime))""") - instance.query("""CREATE TABLE dailyhistory.yellow_tripdata + first.query("""CREATE TABLE dailyhistory.yellow_tripdata ON CLUSTER events AS dailyhistory.yellow_tripdata_staging ENGINE = Distributed('events', 'dailyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") - instance.query("""INSERT INTO dailyhistory.yellow_tripdata + first.query("""INSERT INTO dailyhistory.yellow_tripdata SELECT * FROM generateRandom( 'id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -119,8 +119,8 @@ class Task: 1, 10, 2) LIMIT 50;""") # monthly partition database - instance.query("create database IF NOT EXISTS monthlyhistory on cluster events;") - instance.query("""CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events + first.query("create database IF NOT EXISTS monthlyhistory on cluster events;") + first.query("""CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -153,16 +153,16 @@ class Task: ORDER BY (tpep_pickup_datetime, id) PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime))""") - instance.query("""CREATE TABLE monthlyhistory.yellow_tripdata + first.query("""CREATE TABLE monthlyhistory.yellow_tripdata ON CLUSTER events AS monthlyhistory.yellow_tripdata_staging ENGINE = Distributed('events', 'monthlyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") def check(self): - instance = cluster.instances["first"] - a = TSV(instance.query("SELECT count() from dailyhistory.yellow_tripdata")) - b = TSV(instance.query("SELECT count() from monthlyhistory.yellow_tripdata")) + first = cluster.instances["first"] + a = TSV(first.query("SELECT count() from dailyhistory.yellow_tripdata")) + b = TSV(first.query("SELECT count() from monthlyhistory.yellow_tripdata")) assert a == b, "Distributed tables" for instance_name, instance in cluster.instances.items(): diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index f1e6e9bb5a9..8e375b7b327 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -150,7 +150,7 @@ def test_reload_after_loading(started_cluster): time.sleep(1) # see the comment above replace_in_file_in_container('/etc/clickhouse-server/dictionaries/executable.xml', '82', '83') replace_in_file_in_container('/etc/clickhouse-server/dictionaries/file.txt', '102', '103') - time.sleep(7) + time.sleep(10) assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "103\n" assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "83\n" diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 87e793a7acb..ae0739770ba 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -53,6 +53,7 @@ def test_default_database(test_cluster): def test_create_view(test_cluster): instance = test_cluster.instances['ch3'] + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") test_cluster.ddl_check_query(instance, @@ -76,7 +77,7 @@ def test_on_server_fail(test_cluster): kill_instance.get_docker_handle().stop() request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", - timeout=30) + timeout=180) kill_instance.get_docker_handle().start() test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") @@ -106,11 +107,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout): def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 5) # connection loss will occur only (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 10) # connection loss will occur only (10 sec ZK timeout in config) def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 15) # session should be expired (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 30) # session should be expired (10 sec ZK timeout in config) def test_simple_alters(test_cluster): diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 732816ea224..95d9db27a7d 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -30,6 +30,7 @@ def started_cluster(): def test_chroot_with_same_root(started_cluster): for i, node in enumerate([node1, node2]): + node.query('DROP TABLE IF EXISTS simple SYNC') node.query(''' CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); @@ -44,6 +45,7 @@ def test_chroot_with_same_root(started_cluster): def test_chroot_with_different_root(started_cluster): for i, node in [(1, node1), (3, node3)]: + node.query('DROP TABLE IF EXISTS simple_different SYNC') node.query(''' CREATE TABLE simple_different (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}', date, id, 8192); diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index c0ed4375978..09c15cfd0cf 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -22,6 +22,8 @@ def started_cluster(): cluster.shutdown() def test_identity(started_cluster): + node1.query('DROP TABLE IF EXISTS simple SYNC') + node1.query(''' CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); From bc9bf588c15a3f318c293c7c2384a8a4ddb74605 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 16 Jun 2021 15:34:35 +0300 Subject: [PATCH 276/491] Update 00601_kill_running_query.sh --- tests/queries/0_stateless/00601_kill_running_query.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/00601_kill_running_query.sh b/tests/queries/0_stateless/00601_kill_running_query.sh index 6cb511b7375..3163f8146d0 100755 --- a/tests/queries/0_stateless/00601_kill_running_query.sh +++ b/tests/queries/0_stateless/00601_kill_running_query.sh @@ -11,7 +11,7 @@ function wait_for_query_to_start() while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done } -${CLICKHOUSE_CURL_COMMAND} -q --max-time 30 -sS "$CLICKHOUSE_URL&query_id=test_00601" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k)' > /dev/null & -wait_for_query_to_start 'test_00601' -$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = 'test_00601'" +${CLICKHOUSE_CURL_COMMAND} -q --max-time 30 -sS "$CLICKHOUSE_URL&query_id=test_00601_$CLICKHOUSE_DATABASE" -d 'SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k)' > /dev/null & +wait_for_query_to_start "test_00601_$CLICKHOUSE_DATABASE" +$CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "KILL QUERY WHERE query_id = 'test_00601_$CLICKHOUSE_DATABASE'" wait From 01389a41892b87c1484d850ee8a9bf765a5ede2a Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 16 Jun 2021 15:34:52 +0300 Subject: [PATCH 277/491] Update 00601_kill_running_query.reference --- tests/queries/0_stateless/00601_kill_running_query.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00601_kill_running_query.reference b/tests/queries/0_stateless/00601_kill_running_query.reference index e200d0ebbc2..3917ff89482 100644 --- a/tests/queries/0_stateless/00601_kill_running_query.reference +++ b/tests/queries/0_stateless/00601_kill_running_query.reference @@ -1 +1 @@ -waiting test_00601 default SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k) +waiting test_00601_default default SELECT sum(ignore(*)) FROM (SELECT number % 1000 AS k, groupArray(number) FROM numbers(50000000) GROUP BY k) From 5c6bda62946db6c732f9eee8e3c34a0ad99ce4ae Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 16 Jun 2021 15:57:36 +0300 Subject: [PATCH 278/491] Remove useless srcs --- contrib/arrow-cmake/CMakeLists.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 878b46b39a6..cf4dec16119 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -284,18 +284,6 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/utf8.cc" "${LIBRARY_DIR}/util/value_parsing.cc" - "${LIBRARY_DIR}/vendored/base64.cpp" - "${LIBRARY_DIR}/vendored/datetime/tz.cpp" - "${LIBRARY_DIR}/vendored/double-conversion/bignum.cc" - "${LIBRARY_DIR}/vendored/double-conversion/double-conversion.cc" - "${LIBRARY_DIR}/vendored/double-conversion/bignum-dtoa.cc" - "${LIBRARY_DIR}/vendored/double-conversion/fast-dtoa.cc" - "${LIBRARY_DIR}/vendored/double-conversion/cached-powers.cc" - "${LIBRARY_DIR}/vendored/double-conversion/fixed-dtoa.cc" - "${LIBRARY_DIR}/vendored/double-conversion/diy-fp.cc" - "${LIBRARY_DIR}/vendored/double-conversion/strtod.cc" - - "${LIBRARY_DIR}/c/bridge.cc" ${ORC_SRCS} ) From 81998a5c6f9c952a41f7635a670e1530f5d12d79 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 16 Jun 2021 16:12:23 +0300 Subject: [PATCH 279/491] Fix --- contrib/arrow-cmake/CMakeLists.txt | 1 + contrib/ryu | 1 + 2 files changed, 2 insertions(+) create mode 160000 contrib/ryu diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index cf4dec16119..2237be9913a 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -284,6 +284,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/utf8.cc" "${LIBRARY_DIR}/util/value_parsing.cc" + "${LIBRARY_DIR}/vendored/base64.cpp" ${ORC_SRCS} ) diff --git a/contrib/ryu b/contrib/ryu new file mode 160000 index 00000000000..5b4a853534b --- /dev/null +++ b/contrib/ryu @@ -0,0 +1 @@ +Subproject commit 5b4a853534b47438b4d97935370f6b2397137c2b From 83919363eae17728f81513412d413cfdded9df8b Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 16 Jun 2021 16:55:34 +0300 Subject: [PATCH 280/491] Fix broken contrib --- contrib/ryu | 1 - 1 file changed, 1 deletion(-) delete mode 160000 contrib/ryu diff --git a/contrib/ryu b/contrib/ryu deleted file mode 160000 index 5b4a853534b..00000000000 --- a/contrib/ryu +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5b4a853534b47438b4d97935370f6b2397137c2b From ba08a580f8476f8e993528e8ae8ee8eebe0372f5 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 16 Jun 2021 17:33:14 +0300 Subject: [PATCH 281/491] Add test --- programs/library-bridge/Handlers.cpp | 2 +- programs/odbc-bridge/ColumnInfoHandler.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- programs/odbc-bridge/SchemaAllowedHandler.cpp | 2 +- src/Core/Settings.h | 6 ++--- src/Server/HTTP/HTMLForm.cpp | 22 +++++++-------- src/Server/HTTP/HTMLForm.h | 15 ++++++----- src/Server/HTTP/HTTPServerRequest.cpp | 2 +- src/Server/HTTPHandler.cpp | 2 +- src/Server/InterserverIOHTTPHandler.cpp | 2 +- src/Server/ReplicasStatusHandler.cpp | 2 +- .../0_stateless/01903_http_fields.reference | 8 ++++++ .../queries/0_stateless/01903_http_fields.sh | 27 +++++++++++++++++++ 14 files changed, 66 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/01903_http_fields.reference create mode 100755 tests/queries/0_stateless/01903_http_fields.sh diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index bc955705556..ec82d7d52f4 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -51,7 +51,7 @@ namespace void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { LOG_TRACE(log, "Request URI: {}", request.getURI()); - HTMLForm params(getContext(), request); + HTMLForm params(getContext()->getSettingsRef(), request); if (!params.has("method")) { diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index c968ff12f5b..4b503afb3b3 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -69,7 +69,7 @@ namespace void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(getContext(), request, request.getStream()); + HTMLForm params(getContext()->getSettingsRef(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index 2919519f990..356c2aadee1 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -21,7 +21,7 @@ namespace DB { void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(getContext(), request, request.getStream()); + HTMLForm params(getContext()->getSettingsRef(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index d8aa617c359..edfd2be0dec 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -50,7 +50,7 @@ void ODBCHandler::processError(HTTPServerResponse & response, const std::string void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(getContext(), request); + HTMLForm params(getContext()->getSettingsRef(), request); LOG_TRACE(log, "Request URI: {}", request.getURI()); if (mode == "read") diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index 5bf996ec2b4..af10d8f50cd 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -28,7 +28,7 @@ namespace void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { - HTMLForm params(getContext(), request, request.getStream()); + HTMLForm params(getContext()->getSettingsRef(), request, request.getStream()); LOG_TRACE(log, "Request URI: {}", request.getURI()); auto process_error = [&response, this](const std::string & message) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index fca593be020..3b91e26cd4f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -234,9 +234,9 @@ class IColumn; M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \ M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \ M(UInt64, http_max_uri_size, 1048576, "Maximum URI length of HTTP request", 0) \ - M(UInt64, http_max_fields_number, 1000000, "Maximum number of fields in HTTP header", 0) \ - M(UInt64, http_max_field_name_size, 1024, "Maximum length of field name in HTTP header", 0) \ - M(UInt64, http_max_field_value_size, 8192, "Maximum length of field value in HTTP header", 0) \ + M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \ + M(UInt64, http_max_field_name_size, 1048576, "Maximum length of field name in HTTP header", 0) \ + M(UInt64, http_max_field_value_size, 1048576, "Maximum length of field value in HTTP header", 0) \ M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \ M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ diff --git a/src/Server/HTTP/HTMLForm.cpp b/src/Server/HTTP/HTMLForm.cpp index 67f97153861..86e08f3c8e7 100644 --- a/src/Server/HTTP/HTMLForm.cpp +++ b/src/Server/HTTP/HTMLForm.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include @@ -36,39 +36,39 @@ const std::string HTMLForm::ENCODING_MULTIPART = "multipart/form-data"; const int HTMLForm::UNKNOWN_CONTENT_LENGTH = -1; -HTMLForm::HTMLForm(ContextPtr context) - : max_fields_number(context->getSettingsRef().http_max_fields_number) - , max_field_name_size(context->getSettingsRef().http_max_field_name_size) - , max_field_value_size(context->getSettingsRef().http_max_field_value_size) +HTMLForm::HTMLForm(const Settings & settings) + : max_fields_number(settings.http_max_fields) + , max_field_name_size(settings.http_max_field_name_size) + , max_field_value_size(settings.http_max_field_value_size) , encoding(ENCODING_URL) { } -HTMLForm::HTMLForm(ContextPtr context, const std::string & encoding_) : HTMLForm(context) +HTMLForm::HTMLForm(const Settings & settings, const std::string & encoding_) : HTMLForm(settings) { encoding = encoding_; } -HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler) - : HTMLForm(context) +HTMLForm::HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler) + : HTMLForm(settings) { load(request, requestBody, handler); } -HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody) : HTMLForm(context) +HTMLForm::HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody) : HTMLForm(settings) { load(request, requestBody); } -HTMLForm::HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request) : HTMLForm(context, Poco::URI(request.getURI())) +HTMLForm::HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request) : HTMLForm(settings, Poco::URI(request.getURI())) { } -HTMLForm::HTMLForm(ContextPtr context, const Poco::URI & uri) : HTMLForm(context) +HTMLForm::HTMLForm(const Settings & settings, const Poco::URI & uri) : HTMLForm(settings) { ReadBufferFromString istr(uri.getRawQuery()); // STYLE_CHECK_ALLOW_STD_STRING_STREAM readQuery(istr); diff --git a/src/Server/HTTP/HTMLForm.h b/src/Server/HTTP/HTMLForm.h index 7935f7b53f1..16889b41d80 100644 --- a/src/Server/HTTP/HTMLForm.h +++ b/src/Server/HTTP/HTMLForm.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -13,6 +12,8 @@ namespace DB { +struct Settings; + class HTMLForm : public Poco::Net::NameValueCollection, private boost::noncopyable { public: @@ -25,26 +26,26 @@ public: /// Creates an empty HTMLForm and sets the /// encoding to "application/x-www-form-urlencoded". - explicit HTMLForm(ContextPtr context); + explicit HTMLForm(const Settings & settings); /// Creates an empty HTMLForm that uses the given encoding. /// Encoding must be either "application/x-www-form-urlencoded" (which is the default) or "multipart/form-data". - explicit HTMLForm(ContextPtr context, const std::string & encoding); + explicit HTMLForm(const Settings & settings, const std::string & encoding); /// Creates a HTMLForm from the given HTTP request. /// Uploaded files are passed to the given PartHandler. - HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler); + HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler); /// Creates a HTMLForm from the given HTTP request. /// Uploaded files are silently discarded. - HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody); + HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody); /// Creates a HTMLForm from the given HTTP request. /// The request must be a GET request and the form data must be in the query string (URL encoded). /// For POST requests, you must use one of the constructors taking an additional input stream for the request body. - explicit HTMLForm(ContextPtr context, const Poco::Net::HTTPRequest & request); + explicit HTMLForm(const Settings & settings, const Poco::Net::HTTPRequest & request); - explicit HTMLForm(ContextPtr context, const Poco::URI & uri); + explicit HTMLForm(const Settings & settings, const Poco::URI & uri); template T getParsed(const std::string & key, T default_value) diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index d4359ee0355..788c13557f3 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -17,7 +17,7 @@ namespace DB { HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) : max_uri_size(context->getSettingsRef().http_max_uri_size) - , max_fields_number(context->getSettingsRef().http_max_fields_number) + , max_fields_number(context->getSettingsRef().http_max_fields) , max_field_name_size(context->getSettingsRef().http_max_field_name_size) , max_field_value_size(context->getSettingsRef().http_max_field_value_size) { diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 28105de7111..271c1700daf 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -895,7 +895,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (request.getVersion() == HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); - HTMLForm params(request_context, request); + HTMLForm params(request_context->getSettingsRef(), request); with_stacktrace = params.getParsed("stacktrace", false); /// FIXME: maybe this check is already unnecessary. diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 5329d4f77fd..d76361e856e 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -50,7 +50,7 @@ std::pair InterserverIOHTTPHandler::checkAuthentication(HTTPServer void InterserverIOHTTPHandler::processQuery(HTTPServerRequest & request, HTTPServerResponse & response, Output & used_output) { - HTMLForm params(server.context(), request); + HTMLForm params(server.context()->getSettingsRef(), request); LOG_TRACE(log, "Request URI: {}", request.getURI()); diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 7cb47b85cda..b7dc8c14858 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -26,7 +26,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe { try { - HTMLForm params(getContext(), request); + HTMLForm params(getContext()->getSettingsRef(), request); /// Even if lag is small, output detailed information about the lag. bool verbose = params.get("verbose", "") == "1"; diff --git a/tests/queries/0_stateless/01903_http_fields.reference b/tests/queries/0_stateless/01903_http_fields.reference new file mode 100644 index 00000000000..c18b4e9b082 --- /dev/null +++ b/tests/queries/0_stateless/01903_http_fields.reference @@ -0,0 +1,8 @@ +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01903_http_fields.sh b/tests/queries/0_stateless/01903_http_fields.sh new file mode 100755 index 00000000000..0f2b0df13d9 --- /dev/null +++ b/tests/queries/0_stateless/01903_http_fields.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DEFAULT_MAX_NAME_SIZE=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.settings WHERE name='http_max_field_name_size'") +DEFAULT_MAX_VALUE_SIZE=$($CLICKHOUSE_CLIENT -q "SELECT value FROM system.settings WHERE name='http_max_field_value_size'") + +python3 -c "print('a'*($DEFAULT_MAX_NAME_SIZE-2) + ';')" > $CLICKHOUSE_TMP/short_name.txt +python3 -c "print('a'*($DEFAULT_MAX_NAME_SIZE+1) + ';')" > $CLICKHOUSE_TMP/long_name.txt +python3 -c "print('a'*($DEFAULT_MAX_NAME_SIZE-2) + ': ' + 'b'*($DEFAULT_MAX_VALUE_SIZE-2))" > $CLICKHOUSE_TMP/short_short.txt +python3 -c "print('a'*($DEFAULT_MAX_NAME_SIZE-2) + ': ' + 'b'*($DEFAULT_MAX_VALUE_SIZE+1))" > $CLICKHOUSE_TMP/short_long.txt +python3 -c "print('a'*($DEFAULT_MAX_NAME_SIZE+1) + ': ' + 'b'*($DEFAULT_MAX_VALUE_SIZE-2))" > $CLICKHOUSE_TMP/long_short.txt + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H @$CLICKHOUSE_TMP/short_name.txt -d 'SELECT 1' +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}" -H @$CLICKHOUSE_TMP/long_name.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H @$CLICKHOUSE_TMP/short_short.txt -d 'SELECT 1' +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}" -H @$CLICKHOUSE_TMP/short_long.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}" -H @$CLICKHOUSE_TMP/long_short.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' + +# Session and query settings shouldn't affect the HTTP field's name or value sizes. +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}&http_max_field_name_size=$(($DEFAULT_MAX_NAME_SIZE+10))" -H @$CLICKHOUSE_TMP/long_name.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}&http_max_field_value_size=$(($DEFAULT_MAX_VALUE_SIZE+10))" -H @$CLICKHOUSE_TMP/short_long.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' +${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}&http_max_field_name_size=$(($DEFAULT_MAX_NAME_SIZE+10))" -H @$CLICKHOUSE_TMP/long_short.txt -d 'SELECT 1' 2>&1 | grep -Fc '400 Bad Request' + +# TODO: test that session context doesn't affect these settings either. From d2df23a959d65f45a765a9b7615714b55b291245 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 16 Jun 2021 17:59:58 +0300 Subject: [PATCH 282/491] Function formatDateTime fix code comments (#25334) --- src/Functions/formatDateTime.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index b8d5fe7d281..dc075dea7d0 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -477,8 +477,6 @@ public: { for (auto & instruction : instructions) { - // since right now LUT does not support Int64-values and not format instructions for subsecond parts, - // treat DatTime64 values just as DateTime values by ignoring fractional and casting to UInt32. const auto c = DecimalUtils::split(vec[i], scale); instruction.perform(pos, static_cast(c.whole), time_zone); } From dc46d90f4fbb7f9800b14ff68eb546c010cabccb Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 16 Jun 2021 18:03:11 +0300 Subject: [PATCH 283/491] Apply suggestions from code review Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/data-types/array.md | 2 +- docs/en/sql-reference/data-types/nullable.md | 2 +- docs/ru/sql-reference/data-types/array.md | 8 ++++---- docs/ru/sql-reference/data-types/nullable.md | 7 +++---- docs/ru/sql-reference/data-types/tuple.md | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index a76fb9e2e8c..05800d9826d 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -76,7 +76,7 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception ## Array Size {#array-size} -It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use `sizen-1`, where `n` is the wanted dimension. +It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use `sizeN-1`, where `N` is the wanted dimension. Query: diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 7b43beb65f4..341142f17e8 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -23,7 +23,7 @@ To store `Nullable` type values in a table column, ClickHouse uses a separate fi ## Finding NULL {#finding-null} It is possible to find `NULL` values in a column by using `null` subcolumn without reading the whole column. It returns `1` if the corresponding value is `NULL` and `0` otherwise. - +**Example** Query: ``` sql diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index ec45abac7fb..cf4a7ff98a5 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -5,9 +5,9 @@ toc_title: Array(T) # Array(T) {#data-type-array} -Массив из элементов типа `T`. `T` может любым, в том числе, массивом. Таким образом поддержаны многомерные массивы. +Массив из элементов типа `T`. `T` может любым, в том числе массивом. Таким образом поддерживаются многомерные массивы. -## Создание массива {#sozdanie-massiva} +## Создание массива {#creating-an-array} Массив можно создать с помощью функции: @@ -76,8 +76,8 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception ## Размер массива {#array-size} -Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец `sizen-1`, где `n` — требуемое измерение. - +Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец `sizeN-1`, где `N` — требуемое измерение. +**Пример** Запрос: ```sql diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index f3581108dd4..c7bb9d41d96 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -29,8 +29,8 @@ toc_title: Nullable ## Поиск NULL {#finding-null} -Найти в столбце значения `NULL` можно с помощью подстобца `null`, при этом чтение всего столбца не происходит. Подстолбец возвращает `1`, если соответственное значение равно `NULL`, и `0` в противоположном случае. - +Найти в столбце значения `NULL` можно с помощью подстолбца `null`, при этом весь столбец считывать не требуется. Подстолбец содержит `1`, если соответствующее значение равно `NULL`, и `0` если не равно. +**Пример** Запрос: ``` sql @@ -52,7 +52,7 @@ SELECT n.null FROM nullable; └────────┘ ``` -## Пример использования {#primer-ispolzovaniia} +## Пример использования {#usage-example} ``` sql CREATE TABLE t_null(x Int8, y Nullable(Int8)) ENGINE TinyLog @@ -72,4 +72,3 @@ SELECT x + y from t_null │ 5 │ └────────────┘ ``` - diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index babff343a29..55ddd3a23b1 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -49,7 +49,7 @@ SELECT tuple(1,NULL) AS x, toTypeName(x) ## Адресация элементов кортежа {#addressing-tuple-elements} -Можно считывать элементы кортежей с помощью индексов и имен: +К элементам кортежа можно обращаться по индексу и по имени: ``` sql CREATE TABLE named_tuples (`a` Tuple(s String, i Int64)) ENGINE = Memory; From ce02592219b12cdecfff9ac4190845ce6f0af0af Mon Sep 17 00:00:00 2001 From: George Date: Wed, 16 Jun 2021 18:08:47 +0300 Subject: [PATCH 284/491] Small fixes --- docs/en/sql-reference/data-types/array.md | 2 ++ docs/en/sql-reference/data-types/nullable.md | 2 ++ docs/ru/sql-reference/data-types/array.md | 4 +++- docs/ru/sql-reference/data-types/nullable.md | 4 +++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/data-types/array.md b/docs/en/sql-reference/data-types/array.md index 05800d9826d..0a92a634b6b 100644 --- a/docs/en/sql-reference/data-types/array.md +++ b/docs/en/sql-reference/data-types/array.md @@ -78,6 +78,8 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception It is possible to find the size of an array by using the `size0` subcolumn without reading the whole column. For multi-dimensional arrays you can use `sizeN-1`, where `N` is the wanted dimension. +**Example** + Query: ```sql diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 341142f17e8..de53a47afb7 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -23,7 +23,9 @@ To store `Nullable` type values in a table column, ClickHouse uses a separate fi ## Finding NULL {#finding-null} It is possible to find `NULL` values in a column by using `null` subcolumn without reading the whole column. It returns `1` if the corresponding value is `NULL` and `0` otherwise. + **Example** + Query: ``` sql diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index cf4a7ff98a5..abc16751a79 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -43,7 +43,7 @@ SELECT [1, 2] AS x, toTypeName(x) └───────┴────────────────────┘ ``` -## Особенности работы с типами данных {#osobennosti-raboty-s-tipami-dannykh} +## Особенности работы с типами данных {#working-with-data-types} Максимальный размер массива ограничен одним миллионом элементов. @@ -77,7 +77,9 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception ## Размер массива {#array-size} Узнать размер массива можно с помощью подстолбца `size0` без чтения всего столбца. Для многомерных массивов можно использовать подстолбец `sizeN-1`, где `N` — требуемое измерение. + **Пример** + Запрос: ```sql diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index c7bb9d41d96..84a5d6a796c 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -13,7 +13,7 @@ toc_title: Nullable `NULL` — значение по умолчанию для типа `Nullable`, если в конфигурации сервера ClickHouse не указано иное. -## Особенности хранения {#osobennosti-khraneniia} +## Особенности хранения {#storage-features} Для хранения значения типа `Nullable` ClickHouse использует: @@ -30,7 +30,9 @@ toc_title: Nullable ## Поиск NULL {#finding-null} Найти в столбце значения `NULL` можно с помощью подстолбца `null`, при этом весь столбец считывать не требуется. Подстолбец содержит `1`, если соответствующее значение равно `NULL`, и `0` если не равно. + **Пример** + Запрос: ``` sql From 1d6253b9dcafc27b0d0f51045d4b9fe2d2bd471b Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 16 Jun 2021 18:37:51 +0300 Subject: [PATCH 285/491] hardening --- tests/integration/test_distributed_ddl/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index ae0739770ba..28ebc69bc05 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -99,7 +99,7 @@ def _test_on_connection_losses(test_cluster, zk_timeout): with PartitionManager() as pm: pm.drop_instance_zk_connections(kill_instance) - request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=20) + request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=40) time.sleep(zk_timeout) pm.restore_instance_zk_connections(kill_instance) @@ -107,11 +107,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout): def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 10) # connection loss will occur only (10 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 15) # connection loss will occur only (10 sec ZK timeout in config) def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 30) # session should be expired (10 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 60) # session should be expired (10 sec ZK timeout in config) def test_simple_alters(test_cluster): From b9f50811eadabe39a0b555e6715985e66e1d6137 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:11:34 +0300 Subject: [PATCH 286/491] Update docs/en/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index e520d20e86e..edb35a67eaf 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1281,7 +1281,7 @@ To exchange data with Hadoop, you can use [HDFS table engine](../engines/table-e `Arrow` is Apache Arrow’s "file mode" format. It is designed for in-memory random access. -### Data Types Matching {#data_types-matching-4} +### Data Types Matching {#data_types-matching-arrow} The table below shows supported data types and how they match ClickHouse [data types](../sql-reference/data-types/index.md) in `INSERT` and `SELECT` queries. From 6a5abac8098ae422e27c48f6c81d21ae5c8937ae Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:12:30 +0300 Subject: [PATCH 287/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index a414efb2026..099b475b3d8 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1200,7 +1200,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ `Arrow` — это Apache Arrow's "file mode" формат. Он предназначен для произвольного доступа в памяти. -### Соответствие типов данных {#sootvetstvie-tipov-dannykh-4} +### Соответствие типов данных {#data_types-matching-arrow} Таблица ниже содержит поддерживаемые типы данных и их соответствие [типам данных](../sql-reference/data-types/index.md) ClickHouse для запросов `INSERT` и `SELECT`. @@ -1449,4 +1449,3 @@ $ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum ``` text f9725a22f9191e064120d718e26862a9 - ``` - From c765e393133d3da29c8a73f24eb1f392a4704718 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:13:12 +0300 Subject: [PATCH 288/491] Update docs/en/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index edb35a67eaf..461e2f422be 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1312,7 +1312,7 @@ Unsupported Arrow data types: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `EN The data types of ClickHouse table columns do not have to match the corresponding Arrow data fields. When inserting data, ClickHouse interprets data types according to the table above and then [casts](../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) the data to the data type set for the ClickHouse table column. -### Inserting Data {#inserting-data-3} +### Inserting Data {#inserting-data-arrow} You can insert Arrow data from a file into ClickHouse table by the following command: From 9b96f4c9b09867e53d54470c164ac7f1c656df56 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:13:33 +0300 Subject: [PATCH 289/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 099b475b3d8..0172a62cb75 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1231,7 +1231,7 @@ ClickHouse поддерживает настраиваемую точность Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Arrow. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. -### Вставка данных {#vstavka-dannykh-3} +### Вставка данных {#inserting-data-arrow} Чтобы вставить в ClickHouse данные из файла в формате Arrow, используйте команду следующего вида: From a796c98272432c839c6a8efca974dd50b7ab59e6 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:13:52 +0300 Subject: [PATCH 290/491] Update docs/en/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 461e2f422be..25127b0ea00 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1320,7 +1320,7 @@ You can insert Arrow data from a file into ClickHouse table by the following com $ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" ``` -### Selecting Data {#selecting-data-3} +### Selecting Data {#selecting-data-arrow} You can select data from a ClickHouse table and save them into some file in the Arrow format by the following command: From 23807a87b23d98a713819d71ff92c38c2dfdea8a Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:14:06 +0300 Subject: [PATCH 291/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 0172a62cb75..7b269efdd3f 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1239,7 +1239,7 @@ ClickHouse поддерживает настраиваемую точность $ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" ``` -### Вывод данных {#vyvod-dannykh-3} +### Вывод данных {#selecting-data-arrow} Чтобы получить данные из таблицы ClickHouse и сохранить их в файл формата Arrow, используйте команду следующего вида: From 5c509fa9b782d5866bade5f7a85950b16e4404f6 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:14:53 +0300 Subject: [PATCH 292/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 7b269efdd3f..4918df14ca3 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1174,7 +1174,7 @@ SELECT * FROM topic1_stream; ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. -Неподдержанные типы данных Parquet: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +Неподдерживаемые типы данных Parquet: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. From 7c0b3c58989d0c36cc4f549eb5b199ff23241faa Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:15:07 +0300 Subject: [PATCH 293/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 4918df14ca3..2dede494dd7 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1227,7 +1227,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Arrow `DECIMAL` как `Decimal128`. -Неподдержанные типы данных Arrow: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. +Неподдерживаемые типы данных Arrow: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Arrow. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. From e4972c9167bbabf1a83e5e41480468c6d1ce4227 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:16:00 +0300 Subject: [PATCH 294/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 2dede494dd7..7774cb91fc4 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1225,7 +1225,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. -ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Arrow `DECIMAL` как `Decimal128`. +ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных Arrow `DECIMAL` как `Decimal128`. Неподдерживаемые типы данных Arrow: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. From 43ddc0189d0b4f61a1791af34d548cdd84cfcd3a Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 16 Jun 2021 20:16:41 +0300 Subject: [PATCH 295/491] Update docs/ru/interfaces/formats.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 7774cb91fc4..8c8986b71f5 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1229,7 +1229,7 @@ ClickHouse поддерживает настраиваемую точность Неподдерживаемые типы данных Arrow: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Arrow. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. +Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Arrow. При вставке данных ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. ### Вставка данных {#inserting-data-arrow} From dd0fde544c48a70d2b642b893380db8e50659d3c Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 16 Jun 2021 20:22:11 +0300 Subject: [PATCH 296/491] Update formats.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внес небольшие правки. --- docs/ru/interfaces/formats.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 8c8986b71f5..7780a75a706 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1172,11 +1172,11 @@ SELECT * FROM topic1_stream; Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. -ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. +ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. Неподдерживаемые типы данных Parquet: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. +Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. ### Вставка и выборка данных {#vstavka-i-vyborka-dannykh} @@ -1279,7 +1279,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. -ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При обработке запроса `INSERT`, ClickHouse обрабатывает тип данных ORC `DECIMAL` как `Decimal128`. +ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных ORC `DECIMAL` как `Decimal128`. Неподдерживаемые типы данных ORC: `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. From 3d7100ca60126fe0dec13a90f18515f1e6ee885a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Jun 2021 20:51:16 +0300 Subject: [PATCH 297/491] Disable query_plan_filter_push_down. --- src/Core/Settings.h | 2 +- .../01913_join_push_down_bug.reference | 6 ++++++ .../0_stateless/01913_join_push_down_bug.sql | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01913_join_push_down_bug.reference create mode 100644 tests/queries/0_stateless/01913_join_push_down_bug.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e9379186c0e..e145b928596 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -467,7 +467,7 @@ class IColumn; \ M(Bool, query_plan_enable_optimizations, true, "Apply optimizations to query plan", 0) \ M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ - M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \ + M(Bool, query_plan_filter_push_down, false, "Allow to push down filter by predicate query plan step", 0) \ \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ diff --git a/tests/queries/0_stateless/01913_join_push_down_bug.reference b/tests/queries/0_stateless/01913_join_push_down_bug.reference new file mode 100644 index 00000000000..386bde518ea --- /dev/null +++ b/tests/queries/0_stateless/01913_join_push_down_bug.reference @@ -0,0 +1,6 @@ +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 diff --git a/tests/queries/0_stateless/01913_join_push_down_bug.sql b/tests/queries/0_stateless/01913_join_push_down_bug.sql new file mode 100644 index 00000000000..205d0c28841 --- /dev/null +++ b/tests/queries/0_stateless/01913_join_push_down_bug.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test +( + `t` UInt8, + `flag` UInt8, + `id` UInt8 +) +ENGINE = MergeTree +PARTITION BY t +ORDER BY (t, id) +SETTINGS index_granularity = 8192; + +INSERT INTO test VALUES (1,0,1),(1,0,2),(1,0,3),(1,0,4),(1,0,5),(1,0,6),(1,1,7),(0,0,7); + +SELECT id, flag FROM test t1 +INNER JOIN (SELECT DISTINCT id FROM test) AS t2 ON t1.id = t2.id +WHERE flag = 0 and t = 1 AND id NOT IN (SELECT 1 WHERE 0); + +DROP TABLE IF EXISTS test; From ac2f9dd15aa651412b20cb46a0ed3a2cfe0070e8 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 16 Jun 2021 21:19:05 +0300 Subject: [PATCH 298/491] Updated function dateName before merge --- src/Functions/dateName.cpp | 419 +++++++++--------- src/IO/WriteHelpers.h | 1 + .../0_stateless/01811_datename.reference | 59 +-- tests/queries/0_stateless/01811_datename.sql | 112 +++-- 4 files changed, 294 insertions(+), 297 deletions(-) diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index be4888745fb..9c34b0ae55c 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -1,24 +1,19 @@ -#include +#include + +#include +#include + #include #include #include #include +#include #include #include #include -#include -#include -#include #include -#include - -#include -#include - -#include - namespace DB { namespace ErrorCodes @@ -32,10 +27,25 @@ namespace ErrorCodes namespace { -template struct ActionValueTypeMap {}; -template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; -template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; -template <> struct ActionValueTypeMap { using ActionValueType = Int64; }; +template struct DataTypeToTimeTypeMap {}; + +template <> struct DataTypeToTimeTypeMap +{ + using TimeType = UInt16; +}; + +template <> struct DataTypeToTimeTypeMap +{ + using TimeType = UInt32; +}; + +template <> struct DataTypeToTimeTypeMap +{ + using TimeType = Int64; +}; + +template +using DateTypeToTimeType = typename DataTypeToTimeTypeMap::TimeType; class FunctionDateNameImpl : public IFunction { @@ -61,24 +71,30 @@ public: "Number of arguments for function {} doesn't match: passed {}", getName(), toString(arguments.size())); + if (!WhichDataType(arguments[0].type).isString()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 1 argument of function {}. Must be string", arguments[0].type->getName(), getName()); - if (!WhichDataType(arguments[1].type).isDateOrDateTime()) + + WhichDataType first_argument_type(arguments[1].type); + + if (!(first_argument_type.isDate() || first_argument_type.isDateTime() || first_argument_type.isDateTime64())) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 2 argument of function {}. Must be a date or a date with time", arguments[1].type->getName(), getName()); + if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isString()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of 3 argument of function {}. Must be string", arguments[2].type->getName(), getName()); + return std::make_shared(); } @@ -89,8 +105,9 @@ public: { ColumnPtr res; - if (!((res = executeType(arguments, result_type)) || (res = executeType(arguments, result_type)) - || (res = executeType(arguments, result_type)))) + if (!((res = executeType(arguments, result_type)) + || (res = executeType(arguments, result_type)) + || (res = executeType(arguments, result_type)))) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of function {], must be Date or DateTime.", @@ -107,24 +124,15 @@ public: if (!times) return nullptr; - const ColumnConst * datepart_column = checkAndGetColumnConst(arguments[0].column.get()); - if (!datepart_column) + const ColumnConst * date_part_column = checkAndGetColumnConst(arguments[0].column.get()); + if (!date_part_column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first ('datepart') argument of function {}. Must be constant string.", arguments[0].column->getName(), getName()); - using T = typename ActionValueTypeMap::ActionValueType; - auto datepart_writer = DatePartWriter(); - String datepart = datepart_column->getValue(); - - if (!datepart_writer.isCorrectDatePart(datepart)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Illegal value {} of first ('format') argument of function {}. Check documentation", - datepart, - getName()); + String date_part = date_part_column->getValue(); const DateLUTImpl * time_zone_tmp; if (std::is_same_v || std::is_same_v) @@ -132,203 +140,210 @@ public: else time_zone_tmp = &DateLUT::instance(); - const auto & vec = times->getData(); + const auto & times_data = times->getData(); const DateLUTImpl & time_zone = *time_zone_tmp; UInt32 scale [[maybe_unused]] = 0; if constexpr (std::is_same_v) { - scale = vec.getScale(); + scale = times_data.getScale(); } - auto col_res = ColumnString::create(); - auto & dst_data = col_res->getChars(); - auto & dst_offsets = col_res->getOffsets(); - dst_data.resize(vec.size() * (9 /* longest possible word 'Wednesday' */ + 1 /* zero terminator */)); - dst_offsets.resize(vec.size()); + auto result_column = ColumnString::create(); + auto & result_column_data = result_column->getChars(); + auto & result_column_offsets = result_column->getOffsets(); - auto * begin = reinterpret_cast(dst_data.data()); - auto * pos = begin; + /* longest possible word 'Wednesday' with zero terminator */ + static constexpr size_t longest_word_length = 9 + 1; - for (size_t i = 0; i < vec.size(); ++i) + result_column_data.resize_fill(times_data.size() * longest_word_length); + result_column_offsets.resize(times_data.size()); + + auto * begin = reinterpret_cast(result_column_data.data()); + + WriteBuffer buffer(begin, result_column_data.size()); + + using TimeType = DateTypeToTimeType; + callOnDatePartWriter(date_part, [&](const auto & writer) { - if constexpr (std::is_same_v) + for (size_t i = 0; i < times_data.size(); ++i) { - // since right now LUT does not support Int64-values and not format instructions for subsecond parts, - // treat DatTime64 values just as DateTime values by ignoring fractional and casting to UInt32. - const auto c = DecimalUtils::split(vec[i], scale); - datepart_writer.writeDatePart(pos, datepart, static_cast(c.whole), time_zone); + if constexpr (std::is_same_v) + { + const auto components = DecimalUtils::split(times_data[i], scale); + writer.write(buffer, static_cast(components.whole), time_zone); + } + else + { + writer.write(buffer, times_data[i], time_zone); + } + + /// Null terminator + ++buffer.position(); + result_column_offsets[i] = buffer.position() - begin; } - else - { - datepart_writer.writeDatePart(pos, datepart, vec[i], time_zone); - } - dst_offsets[i] = pos - begin; - ++pos; - } - dst_data.resize(pos - begin); - return col_res; + }); + + result_column_data.resize(buffer.position() - begin); + + return result_column; } private: + template - class DatePartWriter + struct YearWriter { - public: - void writeDatePart(char *& target, const String & datepart, Time source, const DateLUTImpl & timezone) + static void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) { - datepart_functions.at(datepart)(target, source, timezone); - } - - bool isCorrectDatePart(const String & datepart) { return datepart_functions.find(datepart) != datepart_functions.end(); } - - private: - const std::unordered_map datepart_functions = { - {"year", writeYear}, - {"quarter", writeQuarter}, - {"month", writeMonth}, - {"dayofyear", writeDayOfYear}, - {"day", writeDay}, - {"week", writeWeek}, - {"weekday", writeWeekday}, - {"hour", writeHour}, - {"minute", writeMinute}, - {"second", writeSecond}, - }; - - static inline void writeYear(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToYearImpl::execute(source, timezone)); - } - - static inline void writeQuarter(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToQuarterImpl::execute(source, timezone)); - } - - static inline void writeMonth(char *& target, Time source, const DateLUTImpl & timezone) - { - const auto month = ToMonthImpl::execute(source, timezone); - static constexpr std::string_view monthnames[] - = {"January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December"}; - writeString(target, monthnames[month - 1]); - } - - static inline void writeDayOfYear(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToDayOfYearImpl::execute(source, timezone)); - } - - static inline void writeDay(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToDayOfMonthImpl::execute(source, timezone)); - } - - static inline void writeWeek(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToISOWeekImpl::execute(source, timezone)); - } - - static inline void writeWeekday(char *& target, Time source, const DateLUTImpl & timezone) - { - const auto day = ToDayOfWeekImpl::execute(source, timezone); - static constexpr std::string_view daynames[] = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"}; - writeString(target, daynames[day - 1]); - } - - static inline void writeHour(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToHourImpl::execute(source, timezone)); - } - - static inline void writeMinute(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToMinuteImpl::execute(source, timezone)); - } - - static inline void writeSecond(char *& target, Time source, const DateLUTImpl & timezone) - { - writeNumber(target, ToSecondImpl::execute(source, timezone)); - } - - static inline void writeString(char *& target, const std::string_view & value) - { - size_t size = value.size() + 1; /// With zero terminator - memcpy(target, value.data(), size); - target += size; - } - - template - static inline void writeNumber(char *& target, T value) - { - if (value < 10) - { - *target = value + '0'; - target += 2; - *target = '\0'; - } - else if (value < 100) - { - writeNumber2(target, value); - target += 3; - *target = '\0'; - } - else if (value < 1000) - { - writeNumber3(target, value); - target += 4; - *target = '\0'; - } - else if (value < 10000) - { - writeNumber4(target, value); - target += 5; - *target = '\0'; - } - else - { - throw Exception( - "Illegal value of second ('datetime') argument of function dateName. Check documentation.", - ErrorCodes::BAD_ARGUMENTS); - } - } - - template - static inline void writeNumber2(char * p, T v) - { - memcpy(p, &digits100[v * 2], 2); - } - - template - static inline void writeNumber3(char * p, T v) - { - writeNumber2(p, v / 10); - p[2] = v % 10 + '0'; - } - - template - static inline void writeNumber4(char * p, T v) - { - writeNumber2(p, v / 100); - writeNumber2(p + 2, v % 100); + writeText(ToYearImpl::execute(source, timezone), buffer); } }; + + template + struct QuarterWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToQuarterImpl::execute(source, timezone), buffer); + } + }; + + template + struct MonthWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + const auto month = ToMonthImpl::execute(source, timezone); + static constexpr std::string_view month_names[] = + { + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December" + }; + + writeText(month_names[month - 1], buffer); + } + }; + + template + struct WeekWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToISOWeekImpl::execute(source, timezone), buffer); + } + }; + + template + struct DayOfYearWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToDayOfYearImpl::execute(source, timezone), buffer); + } + }; + + template + struct DayWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToDayOfMonthImpl::execute(source, timezone), buffer); + } + }; + + template + struct WeekDayWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + const auto day = ToDayOfWeekImpl::execute(source, timezone); + static constexpr std::string_view day_names[] = + { + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sunday" + }; + + writeText(day_names[day - 1], buffer); + } + }; + + template + struct HourWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToHourImpl::execute(source, timezone), buffer); + } + }; + + template + struct MinuteWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToMinuteImpl::execute(source, timezone), buffer); + } + }; + + template + struct SecondWriter + { + static inline void write(WriteBuffer & buffer, Time source, const DateLUTImpl & timezone) + { + writeText(ToSecondImpl::execute(source, timezone), buffer); + } + }; + + template + void callOnDatePartWriter(const String & date_part, Call && call) const + { + if (date_part == "year") + std::forward(call)(YearWriter