mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge pull request #37232 from zzsmdfj/fix-mysql-datatype-binary0
fix MySQL database engine to compatible with binary(0) dataType
This commit is contained in:
Kruglov Pavel
GitHub
commit
492de1076c
@ -83,7 +83,11 @@ DataTypePtr convertMySQLDataType(MultiEnum<MySQLDataTypesSupport> type_support,
|
||||
res = std::make_shared<DataTypeDate>();
|
||||
}
|
||||
else if (type_name == "binary")
|
||||
{
|
||||
//compatible with binary(0) DataType
|
||||
if (length == 0) length = 1;
|
||||
res = std::make_shared<DataTypeFixedString>(length);
|
||||
}
|
||||
else if (type_name == "datetime" || type_name == "timestamp")
|
||||
{
|
||||
if (!type_support.isSet(MySQLDataTypesSupport::DATETIME64))
|
||||
|
||||
@ -343,9 +343,8 @@ def test_data_types_support_level_for_mysql_database_engine(started_cluster):
|
||||
mysql_node.query("DROP DATABASE test")
|
||||
|
||||
|
||||
# test tool cannot support null by now. TSV format returns \N for null, so cannot compare using == directly
|
||||
# float_values = ['NULL']
|
||||
# float_values = [0] mysql returns 0 while clickhouse returns 0.0, so cannot compare using == directly
|
||||
float_values = [0, "NULL"]
|
||||
clickhouse_float_values = [0, "\\N"]
|
||||
int32_values = [0, 1, -1, 2147483647, -2147483648]
|
||||
uint32_values = [
|
||||
0,
|
||||
@ -357,8 +356,8 @@ int16_values = [0, 1, -1, 32767, -32768]
|
||||
uint16_values = [0, 1, 65535]
|
||||
int8_values = [0, 1, -1, 127, -128]
|
||||
uint8_values = [0, 1, 255]
|
||||
# string_values = ["'ClickHouse'", 'NULL']
|
||||
string_values = ["'ClickHouse'"]
|
||||
string_values = ["'ClickHouse'", "NULL"]
|
||||
clickhouse_string_values = ["ClickHouse", "\\N"]
|
||||
date_values = ["'1970-01-01'"]
|
||||
date2Date32_values = ["'1925-01-01'", "'2283-11-11'"]
|
||||
date2String_values = ["'1000-01-01'", "'9999-12-31'"]
|
||||
@ -381,17 +380,38 @@ timestamp_values = ["'2015-05-18 07:40:01.123'", "'2019-09-16 19:20:11.123'"]
|
||||
timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'"]
|
||||
|
||||
|
||||
def arryToString(expected_clickhouse_values):
|
||||
return "\n".join(str(value) for value in expected_clickhouse_values)
|
||||
|
||||
|
||||
# if expected_clickhouse_values is "", compare MySQL and ClickHouse query results directly
|
||||
@pytest.mark.parametrize(
|
||||
"case_name, mysql_type, expected_ch_type, mysql_values, setting_mysql_datatypes_support_level",
|
||||
"case_name, mysql_type, expected_ch_type, mysql_values, expected_clickhouse_values , setting_mysql_datatypes_support_level",
|
||||
[
|
||||
# test common type mapping
|
||||
# ("common_types", "FLOAT", "Nullable(Float32)", float_values, ""),
|
||||
# ("common_types", "FLOAT UNSIGNED", "Nullable(Float32)", float_values, ""),
|
||||
pytest.param(
|
||||
"common_types",
|
||||
"FLOAT",
|
||||
"Nullable(Float32)",
|
||||
float_values,
|
||||
clickhouse_float_values,
|
||||
"",
|
||||
id="float_1",
|
||||
),
|
||||
pytest.param(
|
||||
"common_types",
|
||||
"FLOAT UNSIGNED",
|
||||
"Nullable(Float32)",
|
||||
float_values,
|
||||
clickhouse_float_values,
|
||||
"",
|
||||
id="float_2",
|
||||
),
|
||||
pytest.param(
|
||||
"common_types",
|
||||
"INT",
|
||||
"Nullable(Int32)",
|
||||
int32_values,
|
||||
int32_values,
|
||||
"",
|
||||
id="common_types_1",
|
||||
),
|
||||
@ -400,6 +420,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT NOT NULL",
|
||||
"Int32",
|
||||
int32_values,
|
||||
int32_values,
|
||||
"",
|
||||
id="common_types_2",
|
||||
),
|
||||
@ -408,6 +429,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT UNSIGNED NOT NULL",
|
||||
"UInt32",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_3",
|
||||
),
|
||||
@ -416,6 +438,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT UNSIGNED",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_4",
|
||||
),
|
||||
@ -424,6 +447,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT UNSIGNED DEFAULT NULL",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_5",
|
||||
),
|
||||
@ -432,6 +456,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT UNSIGNED DEFAULT '1'",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_6",
|
||||
),
|
||||
@ -440,6 +465,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10)",
|
||||
"Nullable(Int32)",
|
||||
int32_values,
|
||||
int32_values,
|
||||
"",
|
||||
id="common_types_7",
|
||||
),
|
||||
@ -448,6 +474,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10) NOT NULL",
|
||||
"Int32",
|
||||
int32_values,
|
||||
int32_values,
|
||||
"",
|
||||
id="common_types_8",
|
||||
),
|
||||
@ -456,6 +483,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10) UNSIGNED NOT NULL",
|
||||
"UInt32",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_8",
|
||||
),
|
||||
@ -464,6 +492,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10) UNSIGNED",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_9",
|
||||
),
|
||||
@ -472,6 +501,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10) UNSIGNED DEFAULT NULL",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_10",
|
||||
),
|
||||
@ -480,6 +510,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INT(10) UNSIGNED DEFAULT '1'",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_11",
|
||||
),
|
||||
@ -488,6 +519,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INTEGER",
|
||||
"Nullable(Int32)",
|
||||
int32_values,
|
||||
int32_values,
|
||||
"",
|
||||
id="common_types_12",
|
||||
),
|
||||
@ -496,6 +528,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"INTEGER UNSIGNED",
|
||||
"Nullable(UInt32)",
|
||||
uint32_values,
|
||||
uint32_values,
|
||||
"",
|
||||
id="common_types_13",
|
||||
),
|
||||
@ -504,6 +537,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"MEDIUMINT",
|
||||
"Nullable(Int32)",
|
||||
mint_values,
|
||||
mint_values,
|
||||
"",
|
||||
id="common_types_14",
|
||||
),
|
||||
@ -512,6 +546,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"MEDIUMINT UNSIGNED",
|
||||
"Nullable(UInt32)",
|
||||
umint_values,
|
||||
umint_values,
|
||||
"",
|
||||
id="common_types_15",
|
||||
),
|
||||
@ -520,6 +555,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"SMALLINT",
|
||||
"Nullable(Int16)",
|
||||
int16_values,
|
||||
int16_values,
|
||||
"",
|
||||
id="common_types_16",
|
||||
),
|
||||
@ -528,6 +564,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"SMALLINT UNSIGNED",
|
||||
"Nullable(UInt16)",
|
||||
uint16_values,
|
||||
uint16_values,
|
||||
"",
|
||||
id="common_types_17",
|
||||
),
|
||||
@ -536,6 +573,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"TINYINT",
|
||||
"Nullable(Int8)",
|
||||
int8_values,
|
||||
int8_values,
|
||||
"",
|
||||
id="common_types_18",
|
||||
),
|
||||
@ -544,6 +582,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"TINYINT UNSIGNED",
|
||||
"Nullable(UInt8)",
|
||||
uint8_values,
|
||||
uint8_values,
|
||||
"",
|
||||
id="common_types_19",
|
||||
),
|
||||
@ -552,6 +591,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"VARCHAR(10)",
|
||||
"Nullable(String)",
|
||||
string_values,
|
||||
clickhouse_string_values,
|
||||
"",
|
||||
id="common_types_20",
|
||||
),
|
||||
@ -561,6 +601,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"Nullable(Date)",
|
||||
date_values,
|
||||
"",
|
||||
"",
|
||||
id="common_types_21",
|
||||
),
|
||||
pytest.param(
|
||||
@ -568,6 +609,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DATE",
|
||||
"Nullable(Date32)",
|
||||
date2Date32_values,
|
||||
"",
|
||||
"date2Date32",
|
||||
id="common_types_22",
|
||||
),
|
||||
@ -576,14 +618,34 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DATE",
|
||||
"Nullable(String)",
|
||||
date2String_values,
|
||||
"",
|
||||
"date2String",
|
||||
id="common_types_23",
|
||||
),
|
||||
pytest.param(
|
||||
"common_types",
|
||||
"binary(1)",
|
||||
"Nullable(FixedString(1))",
|
||||
[1],
|
||||
[1],
|
||||
"",
|
||||
id="common_types_24",
|
||||
),
|
||||
pytest.param(
|
||||
"common_types",
|
||||
"binary(0)",
|
||||
"Nullable(FixedString(1))",
|
||||
["NULL"],
|
||||
["\\N"],
|
||||
"",
|
||||
id="common_types_25",
|
||||
),
|
||||
pytest.param(
|
||||
"decimal_default",
|
||||
"decimal NOT NULL",
|
||||
"Decimal(10, 0)",
|
||||
decimal_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="decimal_1",
|
||||
),
|
||||
@ -592,6 +654,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"decimal",
|
||||
"Nullable(Decimal(10, 0))",
|
||||
decimal_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="decimal_2",
|
||||
),
|
||||
@ -600,6 +663,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"decimal(18, 6) NOT NULL",
|
||||
"Decimal(18, 6)",
|
||||
decimal_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="decimal_3",
|
||||
),
|
||||
@ -608,6 +672,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"decimal(38, 6) NOT NULL",
|
||||
"Decimal(38, 6)",
|
||||
decimal_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="decimal_4",
|
||||
),
|
||||
@ -619,6 +684,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"timestamp",
|
||||
"DateTime",
|
||||
timestamp_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="timestamp_default",
|
||||
),
|
||||
@ -627,6 +693,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"timestamp(6)",
|
||||
"DateTime64(6)",
|
||||
timestamp_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="timestamp_6",
|
||||
),
|
||||
@ -635,6 +702,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DATETIME NOT NULL",
|
||||
"DateTime64(0)",
|
||||
timestamp_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="datetime_default",
|
||||
),
|
||||
@ -643,6 +711,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DATETIME(6) NOT NULL",
|
||||
"DateTime64(6)",
|
||||
timestamp_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="datetime_6_1",
|
||||
),
|
||||
@ -652,6 +721,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"decimal(40, 6) NOT NULL",
|
||||
"String",
|
||||
decimal_values,
|
||||
"",
|
||||
"decimal,datetime64",
|
||||
id="decimal_40_6",
|
||||
),
|
||||
@ -660,6 +730,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"decimal(18, 6) NOT NULL",
|
||||
"String",
|
||||
decimal_values,
|
||||
"",
|
||||
"datetime64",
|
||||
id="decimal_18_6_1",
|
||||
),
|
||||
@ -669,6 +740,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"String",
|
||||
decimal_values,
|
||||
"",
|
||||
"",
|
||||
id="decimal_18_6_2",
|
||||
),
|
||||
pytest.param(
|
||||
@ -676,6 +748,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DATETIME(6) NOT NULL",
|
||||
"DateTime",
|
||||
timestamp_values_no_subsecond,
|
||||
"",
|
||||
"decimal",
|
||||
id="datetime_6_2",
|
||||
),
|
||||
@ -685,6 +758,7 @@ timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'
|
||||
"DateTime",
|
||||
timestamp_values_no_subsecond,
|
||||
"",
|
||||
"",
|
||||
id="datetime_6_3",
|
||||
),
|
||||
],
|
||||
@ -695,6 +769,7 @@ def test_mysql_types(
|
||||
mysql_type,
|
||||
expected_ch_type,
|
||||
mysql_values,
|
||||
expected_clickhouse_values,
|
||||
setting_mysql_datatypes_support_level,
|
||||
):
|
||||
"""Verify that values written to MySQL can be read on ClickHouse side via DB engine MySQL,
|
||||
@ -772,12 +847,18 @@ def test_mysql_types(
|
||||
== expected_ch_type
|
||||
)
|
||||
|
||||
expected_format_clickhouse_values = arryToString(expected_clickhouse_values)
|
||||
if expected_format_clickhouse_values == "":
|
||||
expected_format_clickhouse_values = execute_query(
|
||||
mysql_node, "SELECT value FROM ${mysql_db}.${table_name}"
|
||||
)
|
||||
|
||||
# Validate values
|
||||
assert execute_query(
|
||||
assert expected_format_clickhouse_values == execute_query(
|
||||
clickhouse_node,
|
||||
"SELECT value FROM ${ch_mysql_table}",
|
||||
settings=clickhouse_query_settings,
|
||||
) == execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}")
|
||||
)
|
||||
|
||||
# MySQL DATABASE ENGINE
|
||||
execute_query(
|
||||
@ -800,11 +881,11 @@ def test_mysql_types(
|
||||
)
|
||||
|
||||
# Validate values
|
||||
assert execute_query(
|
||||
assert expected_format_clickhouse_values == execute_query(
|
||||
clickhouse_node,
|
||||
"SELECT value FROM ${ch_mysql_db}.${table_name}",
|
||||
settings=clickhouse_query_settings,
|
||||
) == execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}")
|
||||
)
|
||||
|
||||
# MySQL TABLE FUNCTION
|
||||
# Validate type
|
||||
@ -818,9 +899,7 @@ def test_mysql_types(
|
||||
)
|
||||
|
||||
# Validate values
|
||||
assert execute_query(
|
||||
mysql_node, "SELECT value FROM ${mysql_db}.${table_name}"
|
||||
) == execute_query(
|
||||
assert expected_format_clickhouse_values == execute_query(
|
||||
clickhouse_node,
|
||||
"SELECT value FROM mysql('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')",
|
||||
settings=clickhouse_query_settings,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user