diff --git a/dbms/include/DB/Parsers/ASTCreateQuery.h b/dbms/include/DB/Parsers/ASTCreateQuery.h new file mode 100644 index 00000000000..056603d38e3 --- /dev/null +++ b/dbms/include/DB/Parsers/ASTCreateQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + + +namespace DB +{ + + +/** CREATE TABLE или ATTACH TABLE запрос + */ +class ASTCreateQuery : public IAST +{ +public: + bool attach; /// Запрос ATTACH TABLE, а не CREATE TABLE. + String name; + ASTPtr columns; + ASTPtr storage; + + ASTCreateQuery() {} + ASTCreateQuery(StringRange range_) : IAST(range_), attach(false) {} + + /** Получить текст, который идентифицирует этот элемент. */ + String getID() { return (attach ? "AttachQuery_" : "CreateQuery_") + name; }; +}; + +} diff --git a/dbms/include/DB/Parsers/ASTNameTypePair.h b/dbms/include/DB/Parsers/ASTNameTypePair.h new file mode 100644 index 00000000000..fe4dd176e45 --- /dev/null +++ b/dbms/include/DB/Parsers/ASTNameTypePair.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/** Пара из имени и типа. Например, browser FixedString(2). + */ +class ASTNameTypePair : public IAST +{ +public: + /// имя + String name; + /// тип + ASTPtr type; + + ASTNameTypePair() {} + ASTNameTypePair(StringRange range_) : IAST(range_) {} + + /** Получить текст, который идентифицирует этот элемент. */ + String getID() { return "NameTypePair_" + name; } +}; + +} + diff --git a/dbms/include/DB/Parsers/ExpressionElementParsers.h b/dbms/include/DB/Parsers/ExpressionElementParsers.h index ec60c8027fe..fa2fbe7dbe4 100644 --- a/dbms/include/DB/Parsers/ExpressionElementParsers.h +++ b/dbms/include/DB/Parsers/ExpressionElementParsers.h @@ -37,7 +37,7 @@ protected: }; -/** Функция, например, f(x, y, z) +/** Функция, например, f(x, y + 1, g(z)) */ class ParserFunction : public IParserBase { diff --git a/dbms/include/DB/Parsers/ExpressionListParsers.h b/dbms/include/DB/Parsers/ExpressionListParsers.h index 9613864f3ff..3837c505324 100644 --- a/dbms/include/DB/Parsers/ExpressionListParsers.h +++ b/dbms/include/DB/Parsers/ExpressionListParsers.h @@ -18,6 +18,24 @@ namespace DB typedef std::list > Operators_t; +/** Список элементов, разделённых чем-либо. */ +class ParserList : public IParserBase +{ +public: + ParserList(ParserPtr elem_parser_, ParserPtr separator_parser_, bool allow_empty_ = true) + : elem_parser(elem_parser_), separator_parser(separator_parser_), allow_empty(allow_empty_) + { + } +protected: + String getName() { return "list of elements (" + elem_parser->getName() + ")"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected); +private: + ParserPtr elem_parser; + ParserPtr separator_parser; + bool allow_empty; +}; + + /** Выражение с инфиксным бинарным лево-ассоциативным оператором. * Например, a + b - c + d. * NOTE: если оператор словесный (например, OR), то после него не требуется границы слова. diff --git a/dbms/include/DB/Parsers/ParserCreateQuery.h b/dbms/include/DB/Parsers/ParserCreateQuery.h new file mode 100644 index 00000000000..d31b6ff9f9e --- /dev/null +++ b/dbms/include/DB/Parsers/ParserCreateQuery.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/** Тип или Storage, возможно, параметрический. Например, UInt8 или FixedString(10) или Partitioned(Log, ChunkID) + * Результат парсинга - ASTFunction с параметрами или без. + */ +class ParserIdentifierWithOptionalParameters : public IParserBase +{ +protected: + String getName() { return "identifier with optional parameters"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected); +}; + + +/** Имя и тип через пробел. Например, URL String. */ +class ParserNameTypePair : public IParserBase +{ +protected: + String getName() { return "name and type pair"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected); +}; + + +/** Запрос типа такого: + * CREATE TABLE name + * ( + * name1 type1, + * name2 type2, + * ... + * ) ENGINE = engine + */ +class ParserCreateQuery : public IParserBase +{ +protected: + String getName() { return "CREATE TABLE or ATTACH TABLE query"; } + bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected); +}; + +} diff --git a/dbms/include/DB/Parsers/formatAST.h b/dbms/include/DB/Parsers/formatAST.h index f9631d99c04..8511d54053a 100644 --- a/dbms/include/DB/Parsers/formatAST.h +++ b/dbms/include/DB/Parsers/formatAST.h @@ -5,10 +5,12 @@ #include #include +#include #include #include #include #include +#include namespace DB @@ -19,10 +21,12 @@ namespace DB void formatAST(const IAST & ast, std::ostream & s); void formatAST(const ASTSelectQuery & ast, std::ostream & s); +void formatAST(const ASTCreateQuery & ast, std::ostream & s); void formatAST(const ASTExpressionList & ast, std::ostream & s); void formatAST(const ASTFunction & ast, std::ostream & s); void formatAST(const ASTIdentifier & ast, std::ostream & s); void formatAST(const ASTLiteral & ast, std::ostream & s); +void formatAST(const ASTNameTypePair & ast, std::ostream & s); } diff --git a/dbms/src/Parsers/ExpressionElementParsers.cpp b/dbms/src/Parsers/ExpressionElementParsers.cpp index c4c44357747..b5b6d9676f0 100644 --- a/dbms/src/Parsers/ExpressionElementParsers.cpp +++ b/dbms/src/Parsers/ExpressionElementParsers.cpp @@ -113,7 +113,6 @@ bool ParserIdentifier::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & exp bool ParserFunction::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) { Pos begin = pos; - ASTPtr contents_node; ParserIdentifier id_parser; ParserString open("("), close(")"); @@ -140,7 +139,7 @@ bool ParserFunction::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expec ASTFunction * function_node = new ASTFunction(StringRange(begin, pos)); function_node->name = dynamic_cast(*identifier).name; - function_node->arguments = contents_node; + function_node->arguments = expr_list; function_node->children.push_back(expr_list); node = function_node; return true; diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index 5ce0ad69af2..ddbd86fc0d6 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -11,6 +11,49 @@ namespace DB { + +bool ParserList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) +{ + bool first = true; + ParserWhiteSpaceOrComments ws; + + ASTExpressionList * list = new ASTExpressionList; + node = list; + + while (1) + { + if (first) + { + ASTPtr elem; + if (!elem_parser->parse(pos, end, elem, expected)) + break; + + list->children.push_back(elem); + } + else + { + ws.ignore(pos, end); + if (!separator_parser->ignore(pos, end, expected)) + break; + ws.ignore(pos, end); + + ASTPtr elem; + if (!elem_parser->parse(pos, end, elem, expected)) + return false; + + list->children.push_back(elem); + } + + first = false; + } + + if (!allow_empty && first) + return false; + + return true; +} + + bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) { bool first = true; @@ -154,43 +197,7 @@ ParserAccessExpression::ParserAccessExpression() bool ParserExpressionList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) { - bool first = true; - ParserLogicalOrExpression nested_parser; - ParserWhiteSpaceOrComments ws; - ParserString comma(","); - - ASTExpressionList * p_expr_list = new ASTExpressionList; - ASTExpressionList & expr_list = *p_expr_list; - node = p_expr_list; - - while (1) - { - if (first) - { - ASTPtr elem; - if (!nested_parser.parse(pos, end, elem, expected)) - break; - - expr_list.children.push_back(elem); - } - else - { - ws.ignore(pos, end); - if (!comma.ignore(pos, end, expected)) - break; - ws.ignore(pos, end); - - ASTPtr elem; - if (!nested_parser.parse(pos, end, elem, expected)) - return false; - - expr_list.children.push_back(elem); - } - - first = false; - } - - return true; + return ParserList(new ParserLogicalOrExpression, new ParserString(",")).parse(pos, end, node, expected); } diff --git a/dbms/src/Parsers/ParserCreateQuery.cpp b/dbms/src/Parsers/ParserCreateQuery.cpp new file mode 100644 index 00000000000..0f4b041c2a2 --- /dev/null +++ b/dbms/src/Parsers/ParserCreateQuery.cpp @@ -0,0 +1,155 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ + + +bool ParserIdentifierWithOptionalParameters::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) +{ + ParserIdentifier non_parametric; + ParserFunction parametric; + + Pos begin = pos; + + if (parametric.parse(pos, end, node, expected)) + { + return true; + } + pos = begin; + + ASTPtr ident; + if (non_parametric.parse(pos, end, ident, expected)) + { + ASTFunction * func = new ASTFunction(StringRange(begin, pos)); + node = func; + func->name = dynamic_cast(*ident).name; + return true; + } + pos = begin; + + return false; +} + + +bool ParserNameTypePair::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) +{ + ParserIdentifier name_parser; + ParserIdentifierWithOptionalParameters type_parser; + ParserWhiteSpaceOrComments ws_parser; + + Pos begin = pos; + + ASTPtr name, type; + if (name_parser.parse(pos, end, name, expected) + && ws_parser.ignore(pos, end, expected) + && type_parser.parse(pos, end, type, expected)) + { + ASTNameTypePair * name_type_pair = new ASTNameTypePair(StringRange(begin, pos)); + node = name_type_pair; + name_type_pair->name = dynamic_cast(*name).name; + name_type_pair->type = type; + name_type_pair->children.push_back(type); + return true; + } + + pos = begin; + return false; +} + + +bool ParserCreateQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected) +{ + Pos begin = pos; + + ParserWhiteSpaceOrComments ws; + ParserString s_create("CREATE", true, true); + ParserString s_attach("ATTACH", true, true); + ParserString s_table("TABLE", true, true); + ParserString s_lparen("("); + ParserString s_rparen(")"); + ParserString s_engine("ENGINE", true); + ParserString s_eq("="); + ParserIdentifier name_p; + ParserList columns_p(new ParserNameTypePair, new ParserString(","), false); + ParserIdentifierWithOptionalParameters storage_p; + + ASTPtr name; + ASTPtr columns; + ASTPtr storage; + bool attach = false; + + ws.ignore(pos, end); + + if (!s_create.ignore(pos, end, expected)) + { + if (s_attach.ignore(pos, end, expected)) + attach = true; + else + return false; + } + + ws.ignore(pos, end); + + if (!s_table.ignore(pos, end, expected)) + return false; + + ws.ignore(pos, end); + + if (!name_p.parse(pos, end, name, expected)) + return false; + + ws.ignore(pos, end); + + if (!s_lparen.ignore(pos, end, expected)) + return false; + + ws.ignore(pos, end); + + if (!columns_p.parse(pos, end, columns, expected)) + return false; + + ws.ignore(pos, end); + + if (!s_rparen.ignore(pos, end, expected)) + return false; + + ws.ignore(pos, end); + + if (!s_engine.ignore(pos, end, expected)) + return false; + + ws.ignore(pos, end); + + if (!s_eq.ignore(pos, end, expected)) + return false; + + ws.ignore(pos, end); + + if (!storage_p.parse(pos, end, storage, expected)) + return false; + + ASTCreateQuery * query = new ASTCreateQuery(StringRange(begin, pos)); + node = query; + + query->attach = attach; + query->name = dynamic_cast(*name).name; + query->columns = columns; + query->storage = storage; + + query->children.push_back(columns); + query->children.push_back(storage); + + return true; +} + + +} diff --git a/dbms/src/Parsers/formatAST.cpp b/dbms/src/Parsers/formatAST.cpp index 1bd6cc69ba9..61775b22daa 100644 --- a/dbms/src/Parsers/formatAST.cpp +++ b/dbms/src/Parsers/formatAST.cpp @@ -24,6 +24,13 @@ void formatAST(const IAST & ast, std::ostream & s) formatAST(*select, s); return; } + + const ASTCreateQuery * create = dynamic_cast(&ast); + if (create) + { + formatAST(*create, s); + return; + } const ASTExpressionList * exp_list = dynamic_cast(&ast); if (exp_list) @@ -53,6 +60,13 @@ void formatAST(const IAST & ast, std::ostream & s) return; } + const ASTNameTypePair * ntp = dynamic_cast(&ast); + if (ntp) + { + formatAST(*ntp, s); + return; + } + throw DB::Exception("Unknown element in AST", ErrorCodes::UNKNOWN_ELEMENT_IN_AST); } @@ -62,7 +76,15 @@ void formatAST(const ASTSelectQuery & ast, std::ostream & s) formatAST(*ast.select, s); } -void formatAST(const ASTExpressionList & ast, std::ostream & s) +void formatAST(const ASTCreateQuery & ast, std::ostream & s) +{ + s << (ast.attach ? "ATTACH TABLE " : "CREATE TABLE ") << ast.name << " ("; + formatAST(*ast.columns, s); + s << ") ENGINE = "; + formatAST(*ast.storage, s); +} + +void formatAST(const ASTExpressionList & ast, std::ostream & s) { for (ASTs::const_iterator it = ast.children.begin(); it != ast.children.end(); ++it) { @@ -74,12 +96,16 @@ void formatAST(const ASTExpressionList & ast, std::ostream & s) void formatAST(const ASTFunction & ast, std::ostream & s) { - s << ast.name << '('; - formatAST(*ast.arguments, s); - s << ')'; + s << ast.name; + if (ast.arguments) + { + s << '('; + formatAST(*ast.arguments, s); + s << ')'; + } } -void formatAST(const ASTIdentifier & ast, std::ostream & s) +void formatAST(const ASTIdentifier & ast, std::ostream & s) { s << ast.name; } @@ -89,5 +115,11 @@ void formatAST(const ASTLiteral & ast, std::ostream & s) s << boost::apply_visitor(FieldVisitorToString(), ast.value); } +void formatAST(const ASTNameTypePair & ast, std::ostream & s) +{ + s << ast.name << " "; + formatAST(*ast.type, s); +} + } diff --git a/dbms/src/Parsers/tests/create_parser.cpp b/dbms/src/Parsers/tests/create_parser.cpp new file mode 100644 index 00000000000..4f4ec983679 --- /dev/null +++ b/dbms/src/Parsers/tests/create_parser.cpp @@ -0,0 +1,37 @@ +#include + +#include + +#include +#include +#include + + +int main(int argc, char ** argv) +{ + DB::ParserCreateQuery parser; + DB::ASTPtr ast; + std::string input = "CREATE TABLE hits (URL String, UserAgentMinor2 FixedString(2), EventTime DateTime) ENGINE = Log"; + std::string expected; + + const char * begin = input.data(); + const char * end = begin + input.size(); + const char * pos = begin; + + if (parser.parse(pos, end, ast, expected)) + { + std::cout << "Success." << std::endl; + DB::formatAST(*ast, std::cout); + std::cout << std::endl; + + std::cout << std::endl << ast->getTreeID() << std::endl; + } + else + { + std::cout << "Failed at position " << (pos - begin) << ": " + << mysqlxx::quote << input.substr(pos - begin, 10) + << ", expected " << expected << "." << std::endl; + } + + return 0; +} diff --git a/dbms/src/Parsers/tests/parsers.cpp b/dbms/src/Parsers/tests/select_parser.cpp similarity index 96% rename from dbms/src/Parsers/tests/parsers.cpp rename to dbms/src/Parsers/tests/select_parser.cpp index 2be89032882..e0d4504214b 100644 --- a/dbms/src/Parsers/tests/parsers.cpp +++ b/dbms/src/Parsers/tests/select_parser.cpp @@ -21,7 +21,7 @@ int main(int argc, char ** argv) if (parser.parse(pos, end, ast, expected)) { std::cout << "Success." << std::endl; - DB::formatAST(*ast, std::cout); + DB::formatAST(*ast, std::cerr); std::cout << std::endl; std::cout << std::endl << ast->getTreeID() << std::endl;