fix MySQL COMM_FIELD_LIST response

docker/test/fasttest/run.sh

@@ -259,6 +259,7 @@ function run_tests
         00929_multi_match_edit_distance
         01681_hyperscan_debug_assertion
 
+        01176_mysql_client_interactive          # requires mysql client
         01031_mutations_interpreter_and_context
         01053_ssd_dictionary # this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled
         01083_expressions_in_engine_arguments

docker/test/stateless/Dockerfile

@@ -23,7 +23,8 @@ RUN apt-get update -y \
             telnet \
             tree \
             unixodbc \
-            wget
+            wget \
+            mysql-client-5.7
 
 RUN pip3 install numpy scipy pandas
 

src/Core/MySQL/PacketsProtocolText.cpp

@@ -62,10 +62,10 @@ ColumnDefinition::ColumnDefinition()
 
 ColumnDefinition::ColumnDefinition(
     String schema_, String table_, String org_table_, String name_, String org_name_, uint16_t character_set_, uint32_t column_length_,
-    ColumnType column_type_, uint16_t flags_, uint8_t decimals_)
+    ColumnType column_type_, uint16_t flags_, uint8_t decimals_, bool with_defaults_)
     : schema(std::move(schema_)), table(std::move(table_)), org_table(std::move(org_table_)), name(std::move(name_)),
       org_name(std::move(org_name_)), character_set(character_set_), column_length(column_length_), column_type(column_type_),
-      flags(flags_), decimals(decimals_)
+      flags(flags_), decimals(decimals_), is_comm_field_list_response(with_defaults_)
 {
 }
 
@@ -77,8 +77,15 @@ ColumnDefinition::ColumnDefinition(
 
 size_t ColumnDefinition::getPayloadSize() const
 {
-    return 12 + getLengthEncodedStringSize("def") + getLengthEncodedStringSize(schema) + getLengthEncodedStringSize(table) + getLengthEncodedStringSize(org_table) + \
-            getLengthEncodedStringSize(name) + getLengthEncodedStringSize(org_name) + getLengthEncodedNumberSize(next_length);
+    return 12 +
+           getLengthEncodedStringSize("def") +
+           getLengthEncodedStringSize(schema) +
+           getLengthEncodedStringSize(table) +
+           getLengthEncodedStringSize(org_table) +
+           getLengthEncodedStringSize(name) +
+           getLengthEncodedStringSize(org_name) +
+           getLengthEncodedNumberSize(next_length) +
+           is_comm_field_list_response;
 }
 
 void ColumnDefinition::readPayloadImpl(ReadBuffer & payload)
@@ -115,6 +122,13 @@ void ColumnDefinition::writePayloadImpl(WriteBuffer & buffer) const
     buffer.write(reinterpret_cast<const char *>(&flags), 2);
     buffer.write(reinterpret_cast<const char *>(&decimals), 1);
     writeChar(0x0, 2, buffer);
+    if (is_comm_field_list_response)
+    {
+        /// We should write length encoded int with string size
+        /// followed by string with some "default values" (possibly it's column defaults).
+        /// But we just send NULL for simplicity.
+        writeChar(0xfb, buffer);
+    }
 }
 
 ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex type_index)

src/Core/MySQL/PacketsProtocolText.h

@@ -101,6 +101,9 @@ public:
     ColumnType column_type;
     uint16_t flags;
     uint8_t decimals = 0x00;
+    /// https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition
+    /// There are extra fields in the packet for column defaults
+    bool is_comm_field_list_response = false;
 
 protected:
     size_t getPayloadSize() const override;
@@ -114,7 +117,7 @@ public:
 
     ColumnDefinition(
         String schema_, String table_, String org_table_, String name_, String org_name_, uint16_t character_set_, uint32_t column_length_,
-        ColumnType column_type_, uint16_t flags_, uint8_t decimals_);
+        ColumnType column_type_, uint16_t flags_, uint8_t decimals_, bool with_defaults_ = false);
 
     /// Should be used when column metadata (original name, table, original table, database) is unknown.
     ColumnDefinition(

src/Server/MySQLHandler.cpp

@@ -289,7 +289,7 @@ void MySQLHandler::comFieldList(ReadBuffer & payload)
     for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAll())
     {
         ColumnDefinition column_definition(
-            database, packet.table, packet.table, column.name, column.name, CharacterSet::binary, 100, ColumnType::MYSQL_TYPE_STRING, 0, 0
+            database, packet.table, packet.table, column.name, column.name, CharacterSet::binary, 100, ColumnType::MYSQL_TYPE_STRING, 0, 0, true
         );
         packet_endpoint->sendPacket(column_definition);
     }

tests/queries/0_stateless/01176_mysql_client_interactive.expect

@@ -0,0 +1,26 @@
+#!/usr/bin/expect -f
+
+log_user 0
+set timeout 5
+match_max 100000
+# A default timeout action is to do nothing, change it to fail
+expect_after {
+    timeout {
+        exit 1
+    }
+}
+
+set basedir [file dirname $argv0]
+spawn bash -c "source $basedir/../shell_config.sh ; \$MYSQL_CLIENT_BINARY \$MYSQL_CLIENT_OPT"
+expect "mysql> "
+
+send -- "USE system;\r"
+expect "Database changed"
+
+send -- "SELECT * FROM one;\r"
+expect "| dummy |"
+expect "|     0 |"
+expect "1 row in set"
+
+send -- "quit;\r"
+expect eof

tests/queries/shell_config.sh

@@ -54,6 +54,8 @@ export CLICKHOUSE_PORT_HTTP=${CLICKHOUSE_PORT_HTTP:="8123"}
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=$(${CLICKHOUSE_EXTRACT_CONFIG} --try --key=https_port 2>/dev/null)} 2>/dev/null
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"}
 export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"}
+export CLICKHOUSE_PORT_MYSQL=${CLICKHOUSE_PORT_MYSQL:=$(${CLICKHOUSE_EXTRACT_CONFIG} --try --key=mysql_port 2>/dev/null)} 2>/dev/null
+export CLICKHOUSE_PORT_MYSQL=${CLICKHOUSE_PORT_MYSQL:="9004"}
 
 # Add database and log comment to url params
 if [ -v CLICKHOUSE_URL_PARAMS ]
@@ -87,6 +89,17 @@ export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --ma
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}
 
+export MYSQL_CLIENT_BINARY=${MYSQL_CLIENT_BINARY:="mysql"}
+export MYSQL_CLIENT_CLICKHOUSE_USER=${MYSQL_CLIENT_CLICKHOUSE_USER:="default"}
+# Avoids "Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock'" when connecting to localhost
+[ -v CLICKHOUSE_HOST ] && MYSQL_CLIENT_OPT0+=" --protocol tcp "
+[ -v CLICKHOUSE_HOST ] && MYSQL_CLIENT_OPT0+=" --host ${CLICKHOUSE_HOST} "
+[ -v CLICKHOUSE_PORT_MYSQL ] && MYSQL_CLIENT_OPT0+=" --port ${CLICKHOUSE_PORT_MYSQL} "
+[ -v CLICKHOUSE_DATABASE ] && MYSQL_CLIENT_OPT0+=" --database ${CLICKHOUSE_DATABASE} "
+MYSQL_CLIENT_OPT0+=" --user ${MYSQL_CLIENT_CLICKHOUSE_USER} "
+export MYSQL_CLIENT_OPT="${MYSQL_CLIENT_OPT0:-} ${MYSQL_CLIENT_OPT:-}"
+export MYSQL_CLIENT=${MYSQL_CLIENT:="$MYSQL_CLIENT_BINARY ${MYSQL_CLIENT_OPT:-}"}
+
 function clickhouse_client_removed_host_parameter()
 {
     # removing only `--host=value` and `--host value` (removing '-hvalue' feels to dangerous) with python regex.