thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented MySQL column comments support

Browse Source
This commit is contained in:
Kostiantyn Storozhuk 2021-06-28 21:50:43 +08:00
parent 71603a7d13
commit c2c78929cb
9 changed files with 49 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Databases/MySQL/DatabaseMySQL.cpp
View File

@ -232,7 +232,7 @@ void DatabaseMySQL::fetchLatestTablesStructureIntoCache(
wait_update_tables_name.emplace_back(table_modification_time.first);
}
std::map<String, NamesAndTypesList> tables_and_columns = fetchTablesColumnsList(wait_update_tables_name, local_context);
std::map<String, ColumnsDescription> tables_and_columns = fetchTablesColumnsList(wait_update_tables_name, local_context);
for (const auto & table_and_columns : tables_and_columns)
{
@ -296,7 +296,7 @@ std::map<String, UInt64> DatabaseMySQL::fetchTablesWithModificationTime(ContextP
return tables_with_modification_time;
}
std::map<String, NamesAndTypesList>
std::map<String, ColumnsDescription>
DatabaseMySQL::fetchTablesColumnsList(const std::vector<String> & tables_name, ContextPtr local_context) const
{
const auto & settings = local_context->getSettingsRef();

3
src/Databases/MySQL/DatabaseMySQL.h
View File

@ -8,6 +8,7 @@
#include <Core/MultiEnum.h>
#include <Core/NamesAndTypes.h>
#include <Common/ThreadPool.h>
#include <Storages/ColumnsDescription.h>
#include <Databases/DatabasesCommon.h>
#include <Databases/MySQL/ConnectionMySQLSettings.h>
#include <Parsers/ASTCreateQuery.h>
@ -111,7 +112,7 @@ private:
std::map<String, UInt64> fetchTablesWithModificationTime(ContextPtr local_context) const;
std::map<String, NamesAndTypesList> fetchTablesColumnsList(const std::vector<String> & tables_name, ContextPtr context) const;
std::map<String, ColumnsDescription> fetchTablesColumnsList(const std::vector<String> & tables_name, ContextPtr context) const;
void destroyLocalCacheExtraTables(const std::map<String, UInt64> & tables_with_modification_time) const;

33
src/Databases/MySQL/FetchTablesColumnsList.cpp
View File

@ -40,14 +40,14 @@ String toQueryStringWithQuote(const std::vector<String> & quote_list)
namespace DB
{
std::map<String, NamesAndTypesList> fetchTablesColumnsList(
std::map<String, ColumnsDescription> fetchTablesColumnsList(
mysqlxx::PoolWithFailover & pool,
const String & database_name,
const std::vector<String> & tables_name,
const Settings & settings,
MultiEnum<MySQLDataTypesSupport> type_support)
{
std::map<String, NamesAndTypesList> tables_and_columns;
std::map<String, ColumnsDescription> tables_and_columns;
if (tables_name.empty())
return tables_and_columns;
@ -62,6 +62,7 @@ std::map<String, NamesAndTypesList> fetchTablesColumnsList(
{ std::make_shared<DataTypeUInt64>(), "length" },
{ std::make_shared<DataTypeUInt64>(), "precision" },
{ std::make_shared<DataTypeUInt64>(), "scale" },
{ std::make_shared<DataTypeString>(), "column_comment" },
};
WriteBufferFromOwnString query;
@ -72,8 +73,9 @@ std::map<String, NamesAndTypesList> fetchTablesColumnsList(
" IS_NULLABLE = 'YES' AS is_nullable,"
" COLUMN_TYPE LIKE '%unsigned' AS is_unsigned,"
" CHARACTER_MAXIMUM_LENGTH AS length,"
" NUMERIC_PRECISION as numeric_precision,"
" IF(ISNULL(NUMERIC_SCALE), DATETIME_PRECISION, NUMERIC_SCALE) AS scale" // we know DATETIME_PRECISION as a scale in CH
" NUMERIC_PRECISION AS numeric_precision,"
" IF(ISNULL(NUMERIC_SCALE), DATETIME_PRECISION, NUMERIC_SCALE) AS scale," // we know DATETIME_PRECISION as a scale in CH
" COLUMN_COMMENT AS column_comment"
" FROM INFORMATION_SCHEMA.COLUMNS"
" WHERE ";
@ -94,21 +96,24 @@ std::map<String, NamesAndTypesList> fetchTablesColumnsList(
const auto & char_max_length_col = *block.getByPosition(5).column;
const auto & precision_col = *block.getByPosition(6).column;
const auto & scale_col = *block.getByPosition(7).column;
const auto & column_comment_col = *block.getByPosition(8).column;
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
{
String table_name = table_name_col[i].safeGet<String>();
tables_and_columns[table_name].emplace_back(
column_name_col[i].safeGet<String>(),
convertMySQLDataType(
type_support,
column_type_col[i].safeGet<String>(),
settings.external_table_functions_use_nulls && is_nullable_col[i].safeGet<UInt64>(),
is_unsigned_col[i].safeGet<UInt64>(),
char_max_length_col[i].safeGet<UInt64>(),
precision_col[i].safeGet<UInt64>(),
scale_col[i].safeGet<UInt64>()));
tables_and_columns[table_name].add(
ColumnDescription(
column_name_col[i].safeGet<String>(),
convertMySQLDataType(
type_support,
column_type_col[i].safeGet<String>(),
settings.external_table_functions_use_nulls && is_nullable_col[i].safeGet<UInt64>(),
is_unsigned_col[i].safeGet<UInt64>(),
char_max_length_col[i].safeGet<UInt64>(),
precision_col[i].safeGet<UInt64>(),
scale_col[i].safeGet<UInt64>()),
column_comment_col[i].safeGet<String>()));
}
}
return tables_and_columns;

4
src/Databases/MySQL/FetchTablesColumnsList.h
View File

@ -7,8 +7,8 @@
#include <common/types.h>
#include <Core/MultiEnum.h>
#include <Core/NamesAndTypes.h>
#include <Core/SettingsEnums.h>
#include <Storages/ColumnsDescription.h>
#include <map>
#include <vector>
@ -17,7 +17,7 @@
namespace DB
{
std::map<String, NamesAndTypesList> fetchTablesColumnsList(
std::map<String, ColumnsDescription> fetchTablesColumnsList(
mysqlxx::PoolWithFailover & pool,
const String & database_name,
const std::vector<String> & tables_name,

7
src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
View File

@ -123,7 +123,6 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col
throw Exception("Columns of different size provided.", ErrorCodes::LOGICAL_ERROR);
ColumnsDescription columns_description;
ColumnDescription column_description;
for (
auto [column_name_and_type, declare_column_ast] = std::tuple{columns_name_and_type.begin(), columns_definition->children.begin()};
@ -139,11 +138,7 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col
if (options->changes.count("comment"))
comment = options->changes.at("comment")->as<ASTLiteral>()->value.safeGet<String>();
column_description.name = column_name_and_type->name;
column_description.type = column_name_and_type->type;
if (!comment.empty())
column_description.comment = std::move(comment);
columns_description.add(column_description);
columns_description.add(ColumnDescription(column_name_and_type->name, column_name_and_type->type, comment));
}
return columns_description;

4
src/Storages/ColumnsDescription.cpp
View File

@ -43,8 +43,8 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
ColumnDescription::ColumnDescription(String name_, DataTypePtr type_)
: name(std::move(name_)), type(std::move(type_))
ColumnDescription::ColumnDescription(String name_, DataTypePtr type_, String comment_)
: name(std::move(name_)), type(std::move(type_)), comment(std::move(comment_))
{
}

2
src/Storages/ColumnsDescription.h
View File

@ -39,7 +39,7 @@ struct ColumnDescription
ColumnDescription() = default;
ColumnDescription(ColumnDescription &&) = default;
ColumnDescription(const ColumnDescription &) = default;
ColumnDescription(String name_, DataTypePtr type_);
ColumnDescription(String name_, DataTypePtr type_, String comment_ = "");
bool operator==(const ColumnDescription & other) const;
bool operator!=(const ColumnDescription & other) const { return !(*this == other); }

2
src/TableFunctions/TableFunctionMySQL.cpp
View File

@ -87,7 +87,7 @@ ColumnsDescription TableFunctionMySQL::getActualTableStructure(ContextPtr contex
throw Exception("MySQL table " + (remote_database_name.empty() ? "" : (backQuote(remote_database_name) + "."))
+ backQuote(remote_table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
return ColumnsDescription{columns->second};
return columns->second;
}
StoragePtr TableFunctionMySQL::executeImpl(

19
tests/integration/test_mysql_database_engine/test.py
View File

@ -167,6 +167,25 @@ def test_bad_arguments_for_mysql_database_engine(started_cluster):
assert 'Database engine MySQL requested literal argument.' in str(exception.value)
mysql_node.query("DROP DATABASE test_bad_arguments")
def test_column_comments_for_mysql_database_engine(started_cluster):
with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node:
mysql_node.query("DROP DATABASE IF EXISTS test_database")
mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'")
clickhouse_node.query(
"CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')")
assert 'test_database' in clickhouse_node.query('SHOW DATABASES')
mysql_node.query(
"CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`), `test` int COMMENT 'test comment') ENGINE=InnoDB;")
assert 'test comment' in clickhouse_node.query('DESCRIBE TABLE `test_database`.`test_table`')
mysql_node.query("ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11) COMMENT 'add_column comment'")
assert 'add_column comment' in clickhouse_node.query(
"SELECT comment FROM system.columns WHERE table = 'test_table' AND database = 'test_database'")
mysql_node.query("DROP DATABASE test_database")
def test_data_types_support_level_for_mysql_database_engine(started_cluster):
with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: