From 86c6be4cd961a00ccff5fb05d7c04436acf6d644 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 12 May 2020 17:38:00 +0800 Subject: [PATCH] ISSUES-4006 fix create query convert --- src/Databases/DatabaseFactory.cpp | 2 +- .../MySQL/DatabaseMaterializeMySQL.cpp | 15 +- src/Databases/MySQL/MasterStatusInfo.cpp | 6 + src/Databases/MySQL/MasterStatusInfo.h | 3 - .../MySQL/CreateQueryConvertVisitor.cpp | 282 ++++++++++++++---- .../MySQL/CreateQueryConvertVisitor.h | 37 ++- .../gtest_create_query_convert_visitor.cpp | 218 +++++++------- src/Parsers/MySQL/ASTDeclareIndex.cpp | 4 +- src/Parsers/MySQL/ASTDeclareOption.cpp | 8 +- src/Parsers/MySQL/ASTDeclareTableOptions.cpp | 7 +- .../MySQL/tests/gtest_create_parser.cpp | 9 + 11 files changed, 403 insertions(+), 188 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index cf7e93a0e1f..abfbfbd8878 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -81,7 +81,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String throw Exception("Database engine " + engine_name + " cannot have arguments", ErrorCodes::BAD_ARGUMENTS); if (engine_define->engine->parameters || engine_define->partition_by || engine_define->primary_key || engine_define->order_by || - engine_define->sample_by || engine_define->settings) + engine_define->sample_by || (engine_name != "MySQL" && engine_define->settings)) throw Exception("Database engine " + engine_name + " cannot have parameters, primary_key, order_by, sample_by, settings", ErrorCodes::UNKNOWN_ELEMENT_IN_AST); diff --git a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp index 288e99cd906..178fca8bbe4 100644 --- a/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializeMySQL.cpp @@ -88,7 +88,7 @@ String DatabaseMaterializeMySQL::getCreateQuery(const mysqlxx::Pool::Entry & con { Block show_create_table_header{ {std::make_shared(), "Table"}, - {std::make_shared(), "Create Table"}, + {std::make_shared(), "Create Table"}, }; MySQLBlockInputStream show_create_table( @@ -104,8 +104,12 @@ String DatabaseMaterializeMySQL::getCreateQuery(const mysqlxx::Pool::Entry & con MySQLParser::ParserCreateQuery p_create_query; ASTPtr ast = parseQuery(p_create_query, create_query.data, create_query.data + create_query.size, "", 0, 0); + if (!ast || !ast->as()) + throw Exception("LOGICAL ERROR: ast cannot cast to MySQLParser::ASTCreateQuery.", ErrorCodes::LOGICAL_ERROR); + WriteBufferFromOwnString out; - MySQLVisitor::CreateQueryConvertVisitor::Data data{.out = out}; + ast->as()->database = database; + MySQLVisitor::CreateQueryConvertVisitor::Data data{.out = out, .context = global_context}; MySQLVisitor::CreateQueryConvertVisitor visitor(data); visitor.visit(ast); return out.str(); @@ -152,12 +156,12 @@ void DatabaseMaterializeMySQL::dumpMySQLDatabase() + backQuoteIfNeed(database_name) + " Database */ "; tryToExecuteQuery(query_prefix + " DROP TABLE IF EXISTS " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(dumping_table_name)); - tryToExecuteQuery(query_prefix + getCreateQuery(connection, mysql_database_name, dumping_table_name)); + tryToExecuteQuery(query_prefix + getCreateQuery(connection, database_name, dumping_table_name)); Context context = global_context; context.getClientInfo().query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; context.setCurrentQueryId(""); // generate random query_id - BlockIO streams = executeQuery(query_prefix + " INSERT INTO " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(dumping_table_name), context, true); + BlockIO streams = executeQuery( query_prefix + " INSERT INTO " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(dumping_table_name) + " VALUES", context, true); if (!streams.out) throw Exception("LOGICAL ERROR out stream is undefined.", ErrorCodes::LOGICAL_ERROR); @@ -178,10 +182,11 @@ void DatabaseMaterializeMySQL::synchronization() { std::unique_lock lock{sync_mutex}; - /// Check database is exists in ClickHouse. + LOG_DEBUG(log, "Checking " + database_name + " database status."); while (!sync_quit && !DatabaseCatalog::instance().isDatabaseExist(database_name)) sync_cond.wait_for(lock, std::chrono::seconds(1)); + LOG_DEBUG(log, database_name + " database status is OK."); /// 查找一下位点文件, 如果不存在需要清理目前的数据库, 然后dump全量数据. dumpMySQLDatabase(); } diff --git a/src/Databases/MySQL/MasterStatusInfo.cpp b/src/Databases/MySQL/MasterStatusInfo.cpp index 5c53db22bc4..007f611520f 100644 --- a/src/Databases/MySQL/MasterStatusInfo.cpp +++ b/src/Databases/MySQL/MasterStatusInfo.cpp @@ -2,5 +2,11 @@ namespace DB { +MasterStatusInfo::MasterStatusInfo( + String binlog_file_, UInt64 binlog_position_, String binlog_do_db_, String binlog_ignore_db_, String executed_gtid_set_) + : binlog_file(binlog_file_), binlog_position(binlog_position_), binlog_do_db(binlog_do_db_), binlog_ignore_db(binlog_ignore_db_), + executed_gtid_set(executed_gtid_set_) +{ +} } diff --git a/src/Databases/MySQL/MasterStatusInfo.h b/src/Databases/MySQL/MasterStatusInfo.h index aea50aa3bdd..ba953ceb1dc 100644 --- a/src/Databases/MySQL/MasterStatusInfo.h +++ b/src/Databases/MySQL/MasterStatusInfo.h @@ -20,8 +20,5 @@ struct MasterStatusInfo }; - -std::shared_ptr fetchMasterStatusInfo(mysqlxx::Connection * connection); - } diff --git a/src/Interpreters/MySQL/CreateQueryConvertVisitor.cpp b/src/Interpreters/MySQL/CreateQueryConvertVisitor.cpp index 9c2fe24d48b..c52588cab64 100644 --- a/src/Interpreters/MySQL/CreateQueryConvertVisitor.cpp +++ b/src/Interpreters/MySQL/CreateQueryConvertVisitor.cpp @@ -1,14 +1,20 @@ #include #include #include -#include #include #include #include #include +#include #include #include +#include +#include +#include +#include + + namespace DB { @@ -22,42 +28,7 @@ namespace ErrorCodes namespace MySQLVisitor { -void CreateQueryMatcher::visit(ASTPtr & ast, Data & data) -{ - if (auto * t = ast->as()) - visit(*t, ast, data); -} -void CreateQueryMatcher::visit(ASTCreateQuery & create, ASTPtr & /*ast*/, Data & data) -{ - if (create.like_table) - throw Exception("Cannot convert create like statement to ClickHouse SQL", ErrorCodes::NOT_IMPLEMENTED); - - data.out << "CREATE TABLE " << (create.if_not_exists ? "IF NOT EXISTS" : "") - << (create.database.empty() ? "" : backQuoteIfNeed(create.database) + ".") << backQuoteIfNeed(create.table) << "("; - - if (create.columns_list) - visitColumns(*create.columns_list->as(), create.columns_list, data); - - data.out << ") ENGINE = MergeTree()"; -} - -void CreateQueryMatcher::visitColumns(ASTCreateDefines & create_defines, ASTPtr & /*ast*/, Data & data) -{ - if (!create_defines.columns || create_defines.columns->children.empty()) - throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - - bool is_first = true; - for (auto & column : create_defines.columns->children) - { - if (!is_first) - data.out << ","; - - is_first = false; - visitColumns(*column->as(), column, data); - } -} - -static String convertDataType(const String & type_name, const ASTPtr & arguments, bool is_unsigned, bool /*is_national*/) +static String convertDataType(const String & type_name, const ASTPtr & arguments, bool is_unsigned) { if (type_name == "TINYINT") return is_unsigned ? "UInt8" : "Int8"; @@ -74,7 +45,17 @@ static String convertDataType(const String & type_name, const ASTPtr & arguments else if (type_name == "DOUBLE" || type_name == "PRECISION" || type_name == "REAL") return "Float64"; else if (type_name == "DECIMAL" || type_name == "DEC" || type_name == "NUMERIC" || type_name == "FIXED") - return arguments ? "Decimal(" + queryToString(arguments) + ")" : "Decimal(10, 0)"; + { + if (!arguments) + return "Decimal(10, 0)"; + else if (arguments->children.size() == 1) + return "Decimal(" + queryToString(arguments) + ", 0)"; + else if (arguments->children.size() == 2) + return "Decimal(" + queryToString(arguments) + ")"; + else + throw Exception("Decimal data type family must have exactly two arguments: precision and scale", ErrorCodes::UNKNOWN_TYPE); + } + if (type_name == "DATE") return "Date"; @@ -91,25 +72,74 @@ static String convertDataType(const String & type_name, const ASTPtr & arguments return "String"; } -void CreateQueryMatcher::visitColumns(const ASTDeclareColumn & declare_column, ASTPtr &, Data & data) +static String convertDataType(const String & type_name, const ASTPtr & arguments, bool is_unsigned, bool is_nullable) { - data.out << declare_column.name << " "; + return (is_nullable ? "Nullable(" : "") + convertDataType(type_name, arguments, is_unsigned) + (is_nullable ? ")" : ""); +} +void CreateQueryMatcher::visit(ASTPtr & ast, Data & data) +{ + if (auto * t = ast->as()) + visit(*t, ast, data); +} + +void CreateQueryMatcher::visit(MySQLParser::ASTCreateQuery & create, ASTPtr &, Data & data) +{ + if (create.like_table) + throw Exception("Cannot convert create like statement to ClickHouse SQL", ErrorCodes::NOT_IMPLEMENTED); + + if (create.columns_list) + visit(*create.columns_list->as(), create.columns_list, data); + + if (create.partition_options) + visit(*create.partition_options->as(), create.partition_options, data); + + auto expression_list = std::make_shared(); + expression_list->children = data.primary_keys; + + data.out << "CREATE TABLE " << (create.if_not_exists ? "IF NOT EXISTS" : "") + << (create.database.empty() ? "" : backQuoteIfNeed(create.database) + ".") << backQuoteIfNeed(create.table) + << "(" << queryToString(InterpreterCreateQuery::formatColumns(data.columns_name_and_type)) << ") ENGINE = MergeTree()" + " PARTITION BY " << queryToString(data.getFormattedPartitionByExpression()) + << " ORDER BY " << queryToString(data.getFormattedOrderByExpression()); +} + +void CreateQueryMatcher::visit(MySQLParser::ASTCreateDefines & create_defines, ASTPtr &, Data & data) +{ + if (!create_defines.columns || create_defines.columns->children.empty()) + throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); + + if (create_defines.indices) + { + for (auto & index : create_defines.indices->children) + visit(*index->as(), index, data); + } + + for (auto & column : create_defines.columns->children) + visit(*column->as(), column, data); +} + +void CreateQueryMatcher::visit(const MySQLParser::ASTDeclareIndex & declare_index, ASTPtr &, Data & data) +{ + if (startsWith(declare_index.index_type, "PRIMARY_KEY_")) + data.addPrimaryKey(declare_index.index_columns); +} + +void CreateQueryMatcher::visit(const MySQLParser::ASTDeclareColumn & declare_column, ASTPtr &, Data & data) +{ if (!declare_column.data_type) throw Exception("Missing type in definition of column.", ErrorCodes::UNKNOWN_TYPE); - bool is_unsigned = false; - bool is_nullable = true; - bool is_national = false; + bool is_nullable = true, is_unsigned = false; if (declare_column.column_options) { - if (ASTDeclareOptions * options = declare_column.column_options->as()) + if (MySQLParser::ASTDeclareOptions * options = declare_column.column_options->as()) { if (options->changes.count("is_null")) is_nullable = options->changes["is_null"]->as()->value.safeGet(); - if (options->changes.count("is_national")) - is_national = options->changes["is_national"]->as()->value.safeGet(); + if (options->changes.count("primary_key")) + data.addPrimaryKey(std::make_shared(declare_column.name)); if (options->changes.count("is_unsigned")) is_unsigned = options->changes["is_unsigned"]->as()->value.safeGet(); @@ -119,13 +149,161 @@ void CreateQueryMatcher::visitColumns(const ASTDeclareColumn & declare_column, A } if (ASTFunction * function = declare_column.data_type->as()) - data.out << (is_nullable ? "Nullable(" : "") - << convertDataType(Poco::toUpper(function->name), function->arguments, is_unsigned, is_national) - << (is_nullable ? ")" : ""); + data.columns_name_and_type.emplace_back(declare_column.name, + DataTypeFactory::instance().get(convertDataType(Poco::toUpper(function->name), function->arguments, is_unsigned, is_nullable))); + else if (ASTIdentifier * identifier = declare_column.data_type->as()) + data.columns_name_and_type.emplace_back(declare_column.name, + DataTypeFactory::instance().get(convertDataType(Poco::toUpper(identifier->name), ASTPtr{}, is_unsigned, is_nullable))); + else + throw Exception("Unsupported MySQL data type " + queryToString(declare_column.data_type) + ".", ErrorCodes::NOT_IMPLEMENTED); +} - if (ASTIdentifier * identifier = declare_column.data_type->as()) - data.out << (is_nullable ? "Nullable(" : "") << convertDataType(Poco::toUpper(identifier->name), ASTPtr{}, is_unsigned, is_national) - << (is_nullable ? ")" : ""); +void CreateQueryMatcher::visit(const MySQLParser::ASTDeclarePartitionOptions & declare_partition_options, ASTPtr &, Data & data) +{ + data.addPartitionKey(declare_partition_options.partition_expression); +} + +void CreateQueryMatcher::Data::addPrimaryKey(const ASTPtr & primary_key) +{ + if (const auto & function_index_columns = primary_key->as()) + { + if (function_index_columns->name != "tuple") + throw Exception("Unable to parse function primary key from MySQL.", ErrorCodes::NOT_IMPLEMENTED); + + for (const auto & index_column : function_index_columns->arguments->children) + primary_keys.emplace_back(index_column); + } + else if (const auto & expression_index_columns = primary_key->as()) + { + for (const auto & index_column : expression_index_columns->children) + primary_keys.emplace_back(index_column); + } + else + primary_keys.emplace_back(primary_key); +} + +void CreateQueryMatcher::Data::addPartitionKey(const ASTPtr & partition_key) +{ + if (const auto & function_partition_columns = partition_key->as()) + { + if (function_partition_columns->name != "tuple") + throw Exception("Unable to parse function partition by from MySQL.", ErrorCodes::NOT_IMPLEMENTED); + + for (const auto & partition_column : function_partition_columns->arguments->children) + partition_keys.emplace_back(partition_column); + } + else if (const auto & expression_partition_columns = partition_key->as()) + { + for (const auto & partition_column : expression_partition_columns->children) + partition_keys.emplace_back(partition_column); + } + else + partition_keys.emplace_back(partition_key); +} + +ASTPtr CreateQueryMatcher::Data::getFormattedOrderByExpression() +{ + if (primary_keys.empty()) + return makeASTFunction("tuple"); + + /// TODO: support unique key & key + const auto function = std::make_shared(); + function->name = "tuple"; + function->arguments = std::make_shared(); + function->children.push_back(function->arguments); + function->arguments->children = primary_keys; + + return function; +} + +template +Field choiceBetterRangeSize(TType min, TType max, size_t max_ranges, size_t min_size_pre_range) +{ + UInt64 interval = UInt64(max) - min; + size_t calc_rows_pre_range = std::ceil(interval / double(max_ranges)); + size_t rows_pre_range = std::max(min_size_pre_range, calc_rows_pre_range); + + if (rows_pre_range >= interval) + return Null(); + + return rows_pre_range > std::numeric_limits::max() ? Field(UInt64(rows_pre_range)) : Field(TType(rows_pre_range)); +} + +ASTPtr CreateQueryMatcher::Data::getFormattedPartitionByExpression() +{ + ASTPtr partition_columns = std::make_shared(); + + if (!partition_keys.empty()) + partition_columns->children = partition_keys; + else if (!primary_keys.empty()) + { + ASTPtr expr_list = std::make_shared(); + expr_list->children = primary_keys; + + auto syntax = SyntaxAnalyzer(context).analyze(expr_list, columns_name_and_type); + auto index_expr = ExpressionAnalyzer(expr_list, syntax, context).getActions(false); + const NamesAndTypesList & required_names_and_types = index_expr->getRequiredColumnsWithTypes(); + + const auto & addPartitionColumn = [&](const String & column_name, const DataTypePtr & type, Field better_pre_range_size) + { + partition_columns->children.emplace_back(std::make_shared(column_name)); + + if (type->isNullable()) + partition_columns->children.back() = makeASTFunction("assumeNotNull", partition_columns->children.back()); + + if (!better_pre_range_size.isNull()) + partition_columns->children.back() + = makeASTFunction("divide", partition_columns->children.back(), std::make_shared(better_pre_range_size)); + }; + + for (const auto & required_name_and_type : required_names_and_types) + { + DataTypePtr assume_not_null = required_name_and_type.type; + if (assume_not_null->isNullable()) + assume_not_null = (static_cast(*assume_not_null)).getNestedType(); + + WhichDataType which(assume_not_null); + if (which.isInt8()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isInt16()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isInt32()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isInt64()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isUInt8()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isUInt16()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isUInt32()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isUInt64()) + addPartitionColumn(required_name_and_type.name, required_name_and_type.type, choiceBetterRangeSize( + std::numeric_limits::min(), std::numeric_limits::max(), max_ranges, min_rows_pre_range)); + else if (which.isDateOrDateTime()) + { + partition_columns->children.emplace_back(std::make_shared(required_name_and_type.name)); + + if (required_name_and_type.type->isNullable()) + partition_columns->children.back() = makeASTFunction("assumeNotNull", partition_columns->children.back()); + + partition_columns->children.back() = makeASTFunction("toYYYYMM", partition_columns->children.back()); + } + } + } + + const auto function = std::make_shared(); + function->name = "tuple"; + function->arguments = partition_columns; + function->children.push_back(function->arguments); + return function; } } diff --git a/src/Interpreters/MySQL/CreateQueryConvertVisitor.h b/src/Interpreters/MySQL/CreateQueryConvertVisitor.h index 2c0c9ad6d7f..c4c31f54b88 100644 --- a/src/Interpreters/MySQL/CreateQueryConvertVisitor.h +++ b/src/Interpreters/MySQL/CreateQueryConvertVisitor.h @@ -1,15 +1,16 @@ #pragma once +#include #include #include #include +#include #include +#include namespace DB { -using namespace MySQLParser; - namespace MySQLVisitor { @@ -23,18 +24,42 @@ public: { /// SETTINGS WriteBuffer & out; - std::string declare_columns; + const Context & context; + size_t max_ranges; + size_t min_rows_pre_range; + + ASTs primary_keys; + ASTs partition_keys; + NamesAndTypesList columns_name_and_type; + + void addPrimaryKey(const ASTPtr & primary_key); + + void addPartitionKey(const ASTPtr & partition_key); + + ASTPtr getFormattedOrderByExpression(); + + ASTPtr getFormattedPartitionByExpression(); }; static void visit(ASTPtr & ast, Data & data); static bool needChildVisit(ASTPtr &, const ASTPtr &) { return false; } private: - static void visit(ASTCreateQuery & create, ASTPtr & ast, Data &); + static void visit(MySQLParser::ASTCreateQuery & create, ASTPtr & ast, Data &); - static void visitColumns(ASTCreateDefines & create_defines, ASTPtr & ast, Data & data); + static void visit(MySQLParser::ASTCreateDefines & create_defines, ASTPtr & ast, Data & data); - static void visitColumns(const ASTDeclareColumn & declare_column, ASTPtr & ast, Data & data); + static void visit(const MySQLParser::ASTDeclareIndex & declare_index, ASTPtr & ast, Data & data); + + static void visit(const MySQLParser::ASTDeclareColumn & declare_column, ASTPtr & ast, Data & data); + + static void visit(const MySQLParser::ASTDeclarePartitionOptions & declare_partition_options, ASTPtr & ast, Data & data); + +// static void visitPartitionBy(MySQLParser::ASTCreateQuery & create, ASTPtr & ast, Data & data); + +// static void visitPartitionBy(MySQLParser::ASTCreateDefines & create_defines, ASTPtr & ast, Data & data); + +// static void visitPartitionBy(const MySQLParser::ASTDeclarePartitionOptions & partition_options, ASTPtr & ast, Data & data); // static void visitColumns(const ASTFunction & declare_column, ASTPtr & ast, Data & data); // static void visit(ASTTableJoin & join, const ASTPtr & ast, Data &); diff --git a/src/Interpreters/MySQL/tests/gtest_create_query_convert_visitor.cpp b/src/Interpreters/MySQL/tests/gtest_create_query_convert_visitor.cpp index 6a8ae26e4c4..beed50b1450 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_query_convert_visitor.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_query_convert_visitor.cpp @@ -2,153 +2,149 @@ #include #include #include +#include #include using namespace DB; using namespace MySQLParser; using namespace MySQLVisitor; -TEST(CreateQueryConvert, SimpleNumbersType) +static ContextShared * contextShared() +{ + static SharedContextHolder shared = Context::createShared(); + return shared.get(); +} + +static String convert(const String & input) { ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a tinyint, b SMALLINT, c MEDIUMINT, d INT, e INTEGER, f BIGINT, g DECIMAL, h DEC, i NUMERIC, j " - "FIXED, k FLOAT, l DOUBLE, m DOUBLE PRECISION, n REAL)"; ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; + CreateQueryConvertVisitor::Data data{ + .out = out, .context = Context::createGlobal(contextShared()), .max_ranges = 1000, .min_rows_pre_range = 1000000}; CreateQueryConvertVisitor visitor(data); visitor.visit(ast); + return out.str(); +} + +TEST(CreateQueryConvert, TestConvertNumberColumnsType) +{ + EXPECT_EQ( + convert("CREATE TABLE test(a tinyint, b SMALLINT, c MEDIUMINT, d INT, e INTEGER, f BIGINT, g DECIMAL, h DEC, i NUMERIC, j FIXED, k " + "FLOAT, l DOUBLE, m DOUBLE PRECISION, n REAL)"), + "CREATE TABLE test(`a` Nullable(Int8), `b` Nullable(Int16), `c` Nullable(Int32), `d` Nullable(Int32), `e` Nullable(Int32), `f` " + "Nullable(Int64), `g` Nullable(Decimal(10, 0)), `h` Nullable(Decimal(10, 0)), `i` Nullable(Decimal(10, 0)), `j` " + "Nullable(Decimal(10, 0)), `k` Nullable(Float32), `l` Nullable(Float64), `m` Nullable(Float64), `n` Nullable(Float64)) ENGINE = " + "MergeTree() PARTITION BY tuple() ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(Int8),b Nullable(Int16),c Nullable(Int32),d Nullable(Int32),e Nullable(Int32),f Nullable(Int64),g " - "Nullable(Decimal(10, 0)),h Nullable(Decimal(10, 0)),i Nullable(Decimal(10, 0)),j Nullable(Decimal(10, 0)),k Nullable(Float32),l " - "Nullable(Float64),m Nullable(Float64),n Nullable(Float64)) ENGINE = MergeTree()"); + convert("CREATE TABLE test(a tinyint(1), b SMALLINT(1), c MEDIUMINT(1), d INT(1), e INTEGER(1), f BIGINT(1), g DECIMAL(1), h " + "DEC(2, 1), i NUMERIC(4, 3), j FIXED(6, 5), k FLOAT(1), l DOUBLE(1, 2), m DOUBLE PRECISION(3, 4), n REAL(5, 6))"), + "CREATE TABLE test(`a` Nullable(Int8), `b` Nullable(Int16), `c` Nullable(Int32), `d` Nullable(Int32), `e` Nullable(Int32), `f` " + "Nullable(Int64), `g` Nullable(Decimal(1, 0)), `h` Nullable(Decimal(2, 1)), `i` Nullable(Decimal(4, 3)), `j` Nullable(Decimal(6, " + "5)), `k` Nullable(Float32), `l` Nullable(Float64), `m` Nullable(Float64), `n` Nullable(Float64)) ENGINE = MergeTree() PARTITION " + "BY tuple() ORDER BY tuple()"); + + /// UNSIGNED + EXPECT_EQ( + convert("CREATE TABLE test(a tinyint UNSIGNED, b SMALLINT(1) UNSIGNED, c MEDIUMINT(1) UNSIGNED, d INT(1) UNSIGNED, e INTEGER(1), f " + "BIGINT(1) UNSIGNED, g DECIMAL(1) UNSIGNED, h DEC(2, 1) UNSIGNED, i NUMERIC(4, 3) UNSIGNED, j FIXED(6, 5) UNSIGNED, k FLOAT(1) " + "UNSIGNED, l DOUBLE(1, 2) UNSIGNED, m DOUBLE PRECISION(3, 4) UNSIGNED, n REAL(5, 6) UNSIGNED)"), + "CREATE TABLE test(`a` Nullable(UInt8), `b` Nullable(UInt16), `c` Nullable(UInt32), `d` Nullable(UInt32), `e` Nullable(Int32), `f` " + "Nullable(UInt64), `g` Nullable(Decimal(1, 0)), `h` Nullable(Decimal(2, 1)), `i` Nullable(Decimal(4, 3)), `j` Nullable(Decimal(6, " + "5)), `k` Nullable(Float32), `l` Nullable(Float64), `m` Nullable(Float64), `n` Nullable(Float64)) ENGINE = MergeTree() PARTITION " + "BY tuple() ORDER BY tuple()"); + + /// NOT NULL + EXPECT_EQ( + convert("CREATE TABLE test(a tinyint NOT NULL, b SMALLINT(1) NOT NULL, c MEDIUMINT(1) NOT NULL, d INT(1) NOT NULL, e INTEGER(1), f " + "BIGINT(1) NOT NULL, g DECIMAL(1) NOT NULL, h DEC(2, 1) NOT NULL, i NUMERIC(4, 3) NOT NULL, j FIXED(6, 5) NOT NULL, k FLOAT(1) NOT " + "NULL, l DOUBLE(1, 2) NOT NULL, m DOUBLE PRECISION(3, 4) NOT NULL, n REAL(5, 6) NOT NULL)"), + "CREATE TABLE test(`a` Int8, `b` Int16, `c` Int32, `d` Int32, `e` Nullable(Int32), `f` Int64, `g` Decimal(1, 0), `h` Decimal(2, " + "1), `i` Decimal(4, 3), `j` Decimal(6, 5), `k` Float32, `l` Float64, `m` Float64, `n` Float64) ENGINE = MergeTree() PARTITION BY " + "tuple() ORDER BY tuple()"); + + /// ZEROFILL + EXPECT_EQ( + convert("CREATE TABLE test(a tinyint ZEROFILL, b SMALLINT(1) ZEROFILL, c MEDIUMINT(1) ZEROFILL, d INT(1) ZEROFILL, e INTEGER(1), f " + "BIGINT(1) ZEROFILL, g DECIMAL(1) ZEROFILL, h DEC(2, 1) ZEROFILL, i NUMERIC(4, 3) ZEROFILL, j FIXED(6, 5) ZEROFILL, k FLOAT(1) " + "ZEROFILL, l DOUBLE(1, 2) ZEROFILL, m DOUBLE PRECISION(3, 4) ZEROFILL, n REAL(5, 6) ZEROFILL)"), + "CREATE TABLE test(`a` Nullable(UInt8), `b` Nullable(UInt16), `c` Nullable(UInt32), `d` Nullable(UInt32), `e` Nullable(Int32), `f` " + "Nullable(UInt64), `g` Nullable(Decimal(1, 0)), `h` Nullable(Decimal(2, 1)), `i` Nullable(Decimal(4, 3)), `j` Nullable(Decimal(6, " + "5)), `k` Nullable(Float32), `l` Nullable(Float64), `m` Nullable(Float64), `n` Nullable(Float64)) ENGINE = MergeTree() PARTITION " + "BY tuple() ORDER BY tuple()"); } -TEST(CreateQueryConvert, NumbersTypeWithLength) +TEST(CreateQueryConvert, TestConvertDateTimesColumnsType) { - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a tinyint(1), b SMALLINT(1), c MEDIUMINT(1), d INT(1), e INTEGER(1), f BIGINT(1), g DECIMAL(1), h DEC(1, 2), i NUMERIC(3, 4), j " - "FIXED(5, 6), k FLOAT(1), l DOUBLE(1, 2), m DOUBLE PRECISION(3, 4), n REAL(5, 6))"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + EXPECT_EQ( + convert("CREATE TABLE test(a DATE, b DATETIME, c TIMESTAMP, d TIME, e year)"), + "CREATE TABLE test(`a` Nullable(Date), `b` Nullable(DateTime), `c` Nullable(DateTime), `d` Nullable(DateTime64(3)), `e` " + "Nullable(Int16)) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(Int8),b Nullable(Int16),c Nullable(Int32),d Nullable(Int32),e Nullable(Int32),f Nullable(Int64),g " - "Nullable(Decimal(1)),h Nullable(Decimal(1, 2)),i Nullable(Decimal(3, 4)),j Nullable(Decimal(5, 6)),k Nullable(Float32),l " - "Nullable(Float64),m Nullable(Float64),n Nullable(Float64)) ENGINE = MergeTree()"); -} - -TEST(CreateQueryConvert, NumbersTypeWithUnsigned) -{ - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a tinyint UNSIGNED, b SMALLINT(1) UNSIGNED, c MEDIUMINT(1) UNSIGNED, d INT(1) UNSIGNED, e " - "INTEGER(1), f BIGINT(1) UNSIGNED, g DECIMAL(1) UNSIGNED, h DEC(1, 2) UNSIGNED, i NUMERIC(3, 4) UNSIGNED, j FIXED(5, 6) " - "UNSIGNED, k FLOAT(1) UNSIGNED, l DOUBLE(1, 2) UNSIGNED, m DOUBLE PRECISION(3, 4) UNSIGNED, n REAL(5, 6) UNSIGNED)"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + convert("CREATE TABLE test(a DATE, b DATETIME(1), c TIMESTAMP(1), d TIME(1), e year(4))"), + "CREATE TABLE test(`a` Nullable(Date), `b` Nullable(DateTime), `c` Nullable(DateTime), `d` Nullable(DateTime64(3)), `e` " + "Nullable(Int16)) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(UInt8),b Nullable(UInt16),c Nullable(UInt32),d Nullable(UInt32),e Nullable(Int32),f " - "Nullable(UInt64),g Nullable(Decimal(1)),h Nullable(Decimal(1, 2)),i Nullable(Decimal(3, 4)),j Nullable(Decimal(5, 6)),k " - "Nullable(Float32),l Nullable(Float64),m Nullable(Float64),n Nullable(Float64)) ENGINE = MergeTree()"); + convert( + "CREATE TABLE test(a DATE NOT NULL, b DATETIME(1) NOT NULL, c TIMESTAMP(1) NOT NULL, d TIME(1) NOT NULL, e year(4) NOT NULL)"), + "CREATE TABLE test(`a` Date, `b` DateTime, `c` DateTime, `d` DateTime64(3), `e` Int16) ENGINE = MergeTree() PARTITION BY tuple() " + "ORDER BY tuple()"); } -TEST(CreateQueryConvert, NumbersTypeWithNotNull) +TEST(CreateQueryConvert, TestConvertParitionOptions) { - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a tinyint NOT NULL, b SMALLINT(1) NOT NULL, c MEDIUMINT(1) NOT NULL, d INT(1) NOT NULL, e " - "INTEGER(1), f BIGINT(1) NOT NULL, g DECIMAL(1) NOT NULL, h DEC(1, 2) NOT NULL, i NUMERIC(3, 4) NOT NULL, j FIXED(5, 6) " - "NOT NULL, k FLOAT(1) NOT NULL, l DOUBLE(1, 2) NOT NULL, m DOUBLE PRECISION(3, 4) NOT NULL, n REAL(5, 6) NOT NULL)"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + EXPECT_EQ( + convert("CREATE TABLE test(a DATE NOT NULL) PARTITION BY HASH a"), + "CREATE TABLE test(`a` Date) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Int8,b Int16,c Int32,d Int32,e Nullable(Int32),f Int64,g Decimal(1),h Decimal(1, 2),i Decimal(3, 4),j " - "Decimal(5, 6),k Float32,l Float64,m Float64,n Float64) ENGINE = MergeTree()"); -} - -TEST(CreateQueryConvert, NumbersTypeWithZeroFill) -{ - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a tinyint ZEROFILL, b SMALLINT(1) ZEROFILL, c MEDIUMINT(1) ZEROFILL, d INT(1) ZEROFILL, e " - "INTEGER(1), f BIGINT(1) ZEROFILL, g DECIMAL(1) ZEROFILL, h DEC(1, 2) ZEROFILL, i NUMERIC(3, 4) ZEROFILL, j FIXED(5, 6) " - "ZEROFILL, k FLOAT(1) ZEROFILL, l DOUBLE(1, 2) ZEROFILL, m DOUBLE PRECISION(3, 4) ZEROFILL, n REAL(5, 6) ZEROFILL)"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + convert("CREATE TABLE test(a DATE NOT NULL) PARTITION BY LINEAR HASH a"), + "CREATE TABLE test(`a` Date) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(UInt8),b Nullable(UInt16),c Nullable(UInt32),d Nullable(UInt32),e Nullable(Int32),f " - "Nullable(UInt64),g Nullable(Decimal(1)),h Nullable(Decimal(1, 2)),i Nullable(Decimal(3, 4)),j Nullable(Decimal(5, 6)),k " - "Nullable(Float32),l Nullable(Float64),m Nullable(Float64),n Nullable(Float64)) ENGINE = MergeTree()"); -} - -TEST(CreateQueryConvert, SimpleDateTimesType) -{ - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a DATE, b DATETIME, c TIMESTAMP, d TIME, e year)"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + convert("CREATE TABLE test(a DATE NOT NULL) PARTITION BY RANGE(a)"), + "CREATE TABLE test(`a` Date) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(Date),b Nullable(DateTime),c Nullable(DateTime),d Nullable(DateTime64),e Nullable(Int16)) ENGINE = " - "MergeTree()"); -} - -TEST(CreateQueryConvert, DateTimeTypesWithLength) -{ - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a DATE, b DATETIME(1), c TIMESTAMP(1), d TIME(1), e year(4))"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); - - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + convert("CREATE TABLE test(a DATE NOT NULL, b INT) PARTITION BY RANGE COLUMNS(a, b)"), + "CREATE TABLE test(`a` Date, `b` Nullable(Int32)) ENGINE = MergeTree() PARTITION BY (a, b) ORDER BY tuple()"); EXPECT_EQ( - out.str(), - "CREATE TABLE test(a Nullable(Date),b Nullable(DateTime),c Nullable(DateTime),d Nullable(DateTime64),e Nullable(Int16)) ENGINE = " - "MergeTree()"); + convert("CREATE TABLE test(a DATE NOT NULL) PARTITION BY LIST(a)"), + "CREATE TABLE test(`a` Date) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple()"); + + EXPECT_EQ( + convert("CREATE TABLE test(a DATE NOT NULL, b INT) PARTITION BY LIST COLUMNS(a, b)"), + "CREATE TABLE test(`a` Date, `b` Nullable(Int32)) ENGINE = MergeTree() PARTITION BY (a, b) ORDER BY tuple()"); } -TEST(CreateQueryConvert, DateTimeTypesWithNotNull) +TEST(CreateQueryConvert, TestConvertPrimaryToPartitionBy) { - ParserCreateQuery p_create_query; - String input = "CREATE TABLE test(a DATE NOT NULL, b DATETIME(1) NOT NULL, c TIMESTAMP(1) NOT NULL, d TIME(1) NOT NULL, e year(4) NOT NULL)"; - ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); + EXPECT_EQ(convert("CREATE TABLE test(a DATE NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` Date) ENGINE = MergeTree() PARTITION BY tuple(toYYYYMM(a)) ORDER BY tuple(a)"); - WriteBufferFromOwnString out; - CreateQueryConvertVisitor::Data data{.out = out}; - CreateQueryConvertVisitor visitor(data); - visitor.visit(ast); + EXPECT_EQ(convert("CREATE TABLE test(a DATETIME NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` DateTime) ENGINE = MergeTree() PARTITION BY tuple(toYYYYMM(a)) ORDER BY tuple(a)"); - EXPECT_EQ(out.str(), "CREATE TABLE test(a Date,b DateTime,c DateTime,d DateTime64,e Int16) ENGINE = MergeTree()"); + EXPECT_EQ(convert("CREATE TABLE test(a TINYINT NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` Int8) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple(a)"); + + EXPECT_EQ(convert("CREATE TABLE test(a SMALLINT NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` Int16) ENGINE = MergeTree() PARTITION BY tuple(a) ORDER BY tuple(a)"); + + EXPECT_EQ(convert("CREATE TABLE test(a INT NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` Int32) ENGINE = MergeTree() PARTITION BY tuple(a / 4294968) ORDER BY tuple(a)"); + + EXPECT_EQ(convert("CREATE TABLE test(a BIGINT NOT NULL PRIMARY KEY)"), + "CREATE TABLE test(`a` Int64) ENGINE = MergeTree() PARTITION BY tuple(a / 18446744073709552) ORDER BY tuple(a)"); + + EXPECT_EQ( + convert("CREATE TABLE test(a BIGINT PRIMARY KEY)"), + "CREATE TABLE test(`a` Nullable(Int64)) ENGINE = MergeTree() PARTITION BY tuple(assumeNotNull(a) / 18446744073709552) ORDER BY " + "tuple(a)"); } + diff --git a/src/Parsers/MySQL/ASTDeclareIndex.cpp b/src/Parsers/MySQL/ASTDeclareIndex.cpp index 592e01e23a3..b48656efd71 100644 --- a/src/Parsers/MySQL/ASTDeclareIndex.cpp +++ b/src/Parsers/MySQL/ASTDeclareIndex.cpp @@ -207,7 +207,7 @@ bool ParserDeclareIndex::parseDeclareConstraintIndex(IParser::Pos & pos, String if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_type = temp_node->as()->name; + index_type = "UNIQUE_" + temp_node->as()->name; } } else if (ParserKeyword("PRIMARY KEY").ignore(pos, expected)) @@ -218,7 +218,7 @@ bool ParserDeclareIndex::parseDeclareConstraintIndex(IParser::Pos & pos, String if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_type = temp_node->as()->name; + index_type = "PRIMARY_KEY_" + temp_node->as()->name; } } else if (ParserKeyword("FOREIGN KEY").ignore(pos, expected)) diff --git a/src/Parsers/MySQL/ASTDeclareOption.cpp b/src/Parsers/MySQL/ASTDeclareOption.cpp index 9612d49cb12..188190e0a4f 100644 --- a/src/Parsers/MySQL/ASTDeclareOption.cpp +++ b/src/Parsers/MySQL/ASTDeclareOption.cpp @@ -122,12 +122,10 @@ bool ParserCharsetName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &) while (true) { - if (isWhitespaceASCII(*pos->end)) - break; - else - { + if (!isWhitespaceASCII(*pos->end) && pos->type != TokenType::EndOfStream) ++pos; - } + else + break; } node = std::make_shared(String(begin, pos->end)); diff --git a/src/Parsers/MySQL/ASTDeclareTableOptions.cpp b/src/Parsers/MySQL/ASTDeclareTableOptions.cpp index 066e8c4c414..6d318a20f23 100644 --- a/src/Parsers/MySQL/ASTDeclareTableOptions.cpp +++ b/src/Parsers/MySQL/ASTDeclareTableOptions.cpp @@ -1,11 +1,10 @@ #include #include -#include -#include #include -#include +#include #include +#include #include namespace DB @@ -68,6 +67,8 @@ bool ParserDeclareTableOptions::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp { OptionDescribe("AUTO_INCREMENT", "auto_increment", std::make_shared()), OptionDescribe("AVG_ROW_LENGTH", "avg_row_length", std::make_shared()), + OptionDescribe("CHARSET", "character_set", std::make_shared()), + OptionDescribe("DEFAULT CHARSET", "character_set", std::make_shared()), OptionDescribe("CHARACTER SET", "character_set", std::make_shared()), OptionDescribe("DEFAULT CHARACTER SET", "character_set", std::make_shared()), OptionDescribe("CHECKSUM", "checksum", std::make_shared>()), diff --git a/src/Parsers/MySQL/tests/gtest_create_parser.cpp b/src/Parsers/MySQL/tests/gtest_create_parser.cpp index 5f752c29a7d..1b30a36d84a 100644 --- a/src/Parsers/MySQL/tests/gtest_create_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_create_parser.cpp @@ -31,3 +31,12 @@ TEST(CreateTableParser, SimpleCreate) EXPECT_EQ(ast->as()->table_options->as()->changes["engine"]->as()->name, "INNODB"); EXPECT_TRUE(ast->as()->partition_options->as()); } + +TEST(CreateTableParser, SS) +{ + ParserCreateQuery p_create_query; + String input = "CREATE TABLE `test_table_1` (`a` int DEFAULT NULL, `b` int DEFAULT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"; + ASTPtr ast = parseQuery(p_create_query, input.data(), input.data() + input.size(), "", 0, 0); + ast->dumpTree(std::cerr); + +}