dbms: development [#CONV-2944].

dbms/include/DB/Parsers/ASTCreateQuery.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <DB/Parsers/ASTExpressionList.h>
+#include <DB/Parsers/ASTFunction.h>
+
+
+namespace DB
+{
+
+
+/** CREATE TABLE или ATTACH TABLE запрос
+  */
+class ASTCreateQuery : public IAST
+{
+public:
+	bool attach;	/// Запрос ATTACH TABLE, а не CREATE TABLE.
+	String name;
+	ASTPtr columns;
+	ASTPtr storage;
+
+	ASTCreateQuery() {}
+	ASTCreateQuery(StringRange range_) : IAST(range_), attach(false) {}
+	
+	/** Получить текст, который идентифицирует этот элемент. */
+	String getID() { return (attach ? "AttachQuery_" : "CreateQuery_") + name; };
+};
+
+}

dbms/include/DB/Parsers/ASTNameTypePair.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <DB/Parsers/IAST.h>
+#include <DB/Parsers/ASTFunction.h>
+
+
+namespace DB
+{
+
+/** Пара из имени и типа. Например, browser FixedString(2).
+  */
+class ASTNameTypePair : public IAST
+{
+public:
+	/// имя
+	String name;
+	/// тип
+	ASTPtr type;
+
+    ASTNameTypePair() {}
+    ASTNameTypePair(StringRange range_) : IAST(range_) {}
+
+	/** Получить текст, который идентифицирует этот элемент. */
+	String getID() { return "NameTypePair_" + name; }
+};
+
+}
+

dbms/include/DB/Parsers/ExpressionElementParsers.h

@@ -37,7 +37,7 @@ protected:
 };
 
 
-/** Функция, например, f(x, y, z)
+/** Функция, например, f(x, y + 1, g(z))
   */
 class ParserFunction : public IParserBase
 {

dbms/include/DB/Parsers/ExpressionListParsers.h

@@ -18,6 +18,24 @@ namespace DB
 typedef std::list<std::pair<String, String> > Operators_t;
 
 
+/** Список элементов, разделённых чем-либо. */
+class ParserList : public IParserBase
+{
+public:
+	ParserList(ParserPtr elem_parser_, ParserPtr separator_parser_, bool allow_empty_ = true)
+		: elem_parser(elem_parser_), separator_parser(separator_parser_), allow_empty(allow_empty_)
+	{
+	}
+protected:
+	String getName() { return "list of elements (" + elem_parser->getName() + ")"; }
+	bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected);
+private:
+	ParserPtr elem_parser;
+	ParserPtr separator_parser;
+	bool allow_empty;
+};
+
+
 /** Выражение с инфиксным бинарным лево-ассоциативным оператором.
   * Например, a + b - c + d.
   * NOTE: если оператор словесный (например, OR), то после него не требуется границы слова.

dbms/include/DB/Parsers/ParserCreateQuery.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <DB/Parsers/IParserBase.h>
+#include <DB/Parsers/ExpressionElementParsers.h>
+
+
+namespace DB
+{
+
+/** Тип или Storage, возможно, параметрический. Например, UInt8 или FixedString(10) или Partitioned(Log, ChunkID)
+  * Результат парсинга - ASTFunction с параметрами или без. 
+  */
+class ParserIdentifierWithOptionalParameters : public IParserBase
+{
+protected:
+	String getName() { return "identifier with optional parameters"; }
+	bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected);
+};
+
+
+/** Имя и тип через пробел. Например, URL String. */
+class ParserNameTypePair : public IParserBase
+{
+protected:
+	String getName() { return "name and type pair"; }
+	bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected);
+};
+
+
+/** Запрос типа такого:
+  * CREATE TABLE name
+  * (
+  * 	name1 type1,
+  * 	name2 type2,
+  * 	...
+  * ) ENGINE = engine
+  */
+class ParserCreateQuery : public IParserBase
+{
+protected:
+	String getName() { return "CREATE TABLE or ATTACH TABLE query"; }
+	bool parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected);
+};
+
+}

dbms/include/DB/Parsers/formatAST.h

@@ -5,10 +5,12 @@
 
 #include <DB/Parsers/IAST.h>
 #include <DB/Parsers/ASTSelectQuery.h>
+#include <DB/Parsers/ASTCreateQuery.h>
 #include <DB/Parsers/ASTExpressionList.h>
 #include <DB/Parsers/ASTFunction.h>
 #include <DB/Parsers/ASTIdentifier.h>
 #include <DB/Parsers/ASTLiteral.h>
+#include <DB/Parsers/ASTNameTypePair.h>
 
 
 namespace DB
@@ -19,10 +21,12 @@ namespace DB
 void formatAST(const IAST 				& ast, std::ostream & s);
 
 void formatAST(const ASTSelectQuery 	& ast, std::ostream & s);
+void formatAST(const ASTCreateQuery 	& ast, std::ostream & s);
 void formatAST(const ASTExpressionList 	& ast, std::ostream & s);
 void formatAST(const ASTFunction 		& ast, std::ostream & s);
 void formatAST(const ASTIdentifier 		& ast, std::ostream & s);
 void formatAST(const ASTLiteral 		& ast, std::ostream & s);
+void formatAST(const ASTNameTypePair	& ast, std::ostream & s);
 
 }
 

dbms/src/Parsers/ExpressionElementParsers.cpp

@@ -113,7 +113,6 @@ bool ParserIdentifier::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & exp
 bool ParserFunction::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
 {
 	Pos begin = pos;
-	ASTPtr contents_node;
 
 	ParserIdentifier id_parser;
 	ParserString open("("), close(")");
@@ -140,7 +139,7 @@ bool ParserFunction::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expec
 
 	ASTFunction * function_node = new ASTFunction(StringRange(begin, pos));
 	function_node->name = dynamic_cast<ASTIdentifier &>(*identifier).name;
-	function_node->arguments = contents_node;
+	function_node->arguments = expr_list;
 	function_node->children.push_back(expr_list);
 	node = function_node;
 	return true;

dbms/src/Parsers/ExpressionListParsers.cpp

@@ -11,6 +11,49 @@
 namespace DB
 {
 
+
+bool ParserList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
+{
+	bool first = true;
+	ParserWhiteSpaceOrComments ws;
+
+	ASTExpressionList * list = new ASTExpressionList;
+	node = list;
+
+	while (1)
+	{
+		if (first)
+		{
+			ASTPtr elem;
+			if (!elem_parser->parse(pos, end, elem, expected))
+				break;
+
+			list->children.push_back(elem);
+		}
+		else
+		{
+			ws.ignore(pos, end);
+			if (!separator_parser->ignore(pos, end, expected))
+				break;
+			ws.ignore(pos, end);
+
+			ASTPtr elem;
+			if (!elem_parser->parse(pos, end, elem, expected))
+				return false;
+
+			list->children.push_back(elem);
+		}
+
+		first = false;
+	}
+
+	if (!allow_empty && first)
+		return false;
+
+	return true;
+}
+	
+
 bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
 {
 	bool first = true;
@@ -154,43 +197,7 @@ ParserAccessExpression::ParserAccessExpression()
 
 bool ParserExpressionList::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
 {
-	bool first = true;
-	ParserLogicalOrExpression nested_parser;
-	ParserWhiteSpaceOrComments ws;
-	ParserString comma(",");
-
-	ASTExpressionList * p_expr_list = new ASTExpressionList;
-	ASTExpressionList & expr_list = *p_expr_list;
-	node = p_expr_list;
-
-	while (1)
-	{
-		if (first)
-		{
-			ASTPtr elem;
-			if (!nested_parser.parse(pos, end, elem, expected))
-				break;
-
-			expr_list.children.push_back(elem);
-		}
-		else
-		{
-			ws.ignore(pos, end);
-			if (!comma.ignore(pos, end, expected))
-				break;
-			ws.ignore(pos, end);
-
-			ASTPtr elem;
-			if (!nested_parser.parse(pos, end, elem, expected))
-				return false;
-
-			expr_list.children.push_back(elem);
-		}
-
-		first = false;
-	}
-
-	return true;
+	return ParserList(new ParserLogicalOrExpression, new ParserString(",")).parse(pos, end, node, expected);
 }
 
 

dbms/src/Parsers/ParserCreateQuery.cpp

@@ -0,0 +1,155 @@
+#include <DB/Parsers/ASTFunction.h>
+#include <DB/Parsers/ASTIdentifier.h>
+#include <DB/Parsers/ASTNameTypePair.h>
+#include <DB/Parsers/ASTExpressionList.h>
+#include <DB/Parsers/ASTCreateQuery.h>
+
+#include <DB/Parsers/CommonParsers.h>
+#include <DB/Parsers/ExpressionListParsers.h>
+#include <DB/Parsers/ParserCreateQuery.h>
+
+
+namespace DB
+{
+
+
+bool ParserIdentifierWithOptionalParameters::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
+{
+	ParserIdentifier non_parametric;
+	ParserFunction parametric;
+	
+	Pos begin = pos;
+
+	if (parametric.parse(pos, end, node, expected))
+	{
+		return true;
+	}
+	pos = begin;
+
+	ASTPtr ident;
+	if (non_parametric.parse(pos, end, ident, expected))
+	{
+		ASTFunction * func = new ASTFunction(StringRange(begin, pos));
+		node = func;
+		func->name = dynamic_cast<ASTIdentifier &>(*ident).name;
+		return true;
+	}
+	pos = begin;
+
+	return false;
+}
+
+
+bool ParserNameTypePair::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
+{
+	ParserIdentifier name_parser;
+	ParserIdentifierWithOptionalParameters type_parser;
+	ParserWhiteSpaceOrComments ws_parser;
+	
+	Pos begin = pos;
+
+	ASTPtr name, type;
+	if (name_parser.parse(pos, end, name, expected)
+		&& ws_parser.ignore(pos, end, expected)
+		&& type_parser.parse(pos, end, type, expected))
+	{
+		ASTNameTypePair * name_type_pair = new ASTNameTypePair(StringRange(begin, pos));
+		node = name_type_pair;
+		name_type_pair->name = dynamic_cast<ASTIdentifier &>(*name).name;
+		name_type_pair->type = type;
+		name_type_pair->children.push_back(type);
+		return true;
+	}
+	
+	pos = begin;
+	return false;
+}
+
+
+bool ParserCreateQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, String & expected)
+{
+	Pos begin = pos;
+
+	ParserWhiteSpaceOrComments ws;
+	ParserString s_create("CREATE", true, true);
+	ParserString s_attach("ATTACH", true, true);
+	ParserString s_table("TABLE", true, true);
+	ParserString s_lparen("(");
+	ParserString s_rparen(")");
+	ParserString s_engine("ENGINE", true);
+	ParserString s_eq("=");
+	ParserIdentifier name_p;
+	ParserList columns_p(new ParserNameTypePair, new ParserString(","), false);
+	ParserIdentifierWithOptionalParameters storage_p;
+
+	ASTPtr name;
+	ASTPtr columns;
+	ASTPtr storage;
+	bool attach = false;
+
+	ws.ignore(pos, end);
+
+	if (!s_create.ignore(pos, end, expected))
+	{
+		if (s_attach.ignore(pos, end, expected))
+			attach = true;
+		else
+			return false;
+	}
+
+	ws.ignore(pos, end);
+
+	if (!s_table.ignore(pos, end, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!name_p.parse(pos, end, name, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!s_lparen.ignore(pos, end, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!columns_p.parse(pos, end, columns, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!s_rparen.ignore(pos, end, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!s_engine.ignore(pos, end, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!s_eq.ignore(pos, end, expected))
+		return false;
+
+	ws.ignore(pos, end);
+
+	if (!storage_p.parse(pos, end, storage, expected))
+		return false;
+
+	ASTCreateQuery * query = new ASTCreateQuery(StringRange(begin, pos));
+	node = query;
+
+	query->attach = attach;
+	query->name = dynamic_cast<ASTIdentifier &>(*name).name;
+	query->columns = columns;
+	query->storage = storage;
+
+	query->children.push_back(columns);
+	query->children.push_back(storage);
+
+	return true;
+}
+
+
+}

dbms/src/Parsers/formatAST.cpp

@@ -25,6 +25,13 @@ void formatAST(const IAST & ast, std::ostream & s)
 		return;
 	}
 
+	const ASTCreateQuery * create = dynamic_cast<const ASTCreateQuery *>(&ast);
+	if (create)
+	{
+		formatAST(*create, s);
+		return;
+	}
+	
 	const ASTExpressionList * exp_list = dynamic_cast<const ASTExpressionList *>(&ast);
 	if (exp_list)
 	{
@@ -53,6 +60,13 @@ void formatAST(const IAST & ast, std::ostream & s)
 		return;
 	}
 
+	const ASTNameTypePair * ntp = dynamic_cast<const ASTNameTypePair *>(&ast);
+	if (ntp)
+	{
+		formatAST(*ntp, s);
+		return;
+	}
+
 	throw DB::Exception("Unknown element in AST", ErrorCodes::UNKNOWN_ELEMENT_IN_AST);
 }
 
@@ -62,7 +76,15 @@ void formatAST(const ASTSelectQuery 		& ast, std::ostream & s)
 	formatAST(*ast.select, s);
 }
 
-void formatAST(const ASTExpressionList 	& ast, std::ostream & s)
+void formatAST(const ASTCreateQuery 		& ast, std::ostream & s)
+{
+	s << (ast.attach ? "ATTACH TABLE " : "CREATE TABLE ") << ast.name << " (";
+	formatAST(*ast.columns, s);
+	s << ") ENGINE = ";
+	formatAST(*ast.storage, s);
+}
+
+void formatAST(const ASTExpressionList 		& ast, std::ostream & s)
 {
 	for (ASTs::const_iterator it = ast.children.begin(); it != ast.children.end(); ++it)
 	{
@@ -74,12 +96,16 @@ void formatAST(const ASTExpressionList 	& ast, std::ostream & s)
 
 void formatAST(const ASTFunction 			& ast, std::ostream & s)
 {
-	s << ast.name << '(';
-	formatAST(*ast.arguments, s);
-	s << ')';
+	s << ast.name;
+	if (ast.arguments)
+	{
+		s << '(';
+		formatAST(*ast.arguments, s);
+		s << ')';
+	}
 }
 
-void formatAST(const ASTIdentifier 		& ast, std::ostream & s)
+void formatAST(const ASTIdentifier 			& ast, std::ostream & s)
 {
 	s << ast.name;
 }
@@ -89,5 +115,11 @@ void formatAST(const ASTLiteral 			& ast, std::ostream & s)
 	s << boost::apply_visitor(FieldVisitorToString(), ast.value);
 }
 
+void formatAST(const ASTNameTypePair		& ast, std::ostream & s)
+{
+	s << ast.name << " ";
+	formatAST(*ast.type, s);
+}
+
 }
 

dbms/src/Parsers/tests/create_parser.cpp

@@ -0,0 +1,37 @@
+#include <iostream>
+
+#include <mysqlxx/mysqlxx.h>
+
+#include <DB/Parsers/ASTCreateQuery.h>
+#include <DB/Parsers/ParserCreateQuery.h>
+#include <DB/Parsers/formatAST.h>
+
+
+int main(int argc, char ** argv)
+{
+	DB::ParserCreateQuery parser;
+	DB::ASTPtr ast;
+	std::string input = "CREATE TABLE hits (URL String, UserAgentMinor2 FixedString(2), EventTime DateTime) ENGINE = Log";
+	std::string expected;
+
+	const char * begin = input.data();
+	const char * end = begin + input.size();
+	const char * pos = begin;
+
+	if (parser.parse(pos, end, ast, expected))
+	{
+		std::cout << "Success." << std::endl;
+		DB::formatAST(*ast, std::cout);
+		std::cout << std::endl;
+
+		std::cout << std::endl << ast->getTreeID() << std::endl;
+	}
+	else
+	{
+		std::cout << "Failed at position " << (pos - begin) << ": "
+			<< mysqlxx::quote << input.substr(pos - begin, 10)
+			<< ", expected " << expected << "." << std::endl;
+	}
+
+	return 0;
+}

dbms/src/Parsers/tests/select_parser.cpp

@@ -21,7 +21,7 @@ int main(int argc, char ** argv)
 	if (parser.parse(pos, end, ast, expected))
 	{
 		std::cout << "Success." << std::endl;
-		DB::formatAST(*ast, std::cout);
+		DB::formatAST(*ast, std::cerr);
 		std::cout << std::endl;
 
 		std::cout << std::endl << ast->getTreeID() << std::endl;