Merge pull request #65302 from ClickHouse/backport/24.3/65170

Backport #65170 to 24.3: Rework the tuple serialization/deserialization in CSV format.

src/Core/Settings.h

@@ -981,6 +981,8 @@ class IColumn;
     M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \
     M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \
     M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \
+    M(Bool, output_format_csv_serialize_tuple_into_separate_columns, true, "If it set to true, then Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost)", 0) \
+    M(Bool, input_format_csv_deserialize_separate_columns_into_tuple, true, "If it set to true, then separate columns written in CSV format can be deserialized to Tuple column.", 0) \
     M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \
     M(Bool, input_format_csv_allow_cr_end_of_line, false, "If it is set true, \\r will be allowed at end of line not followed by \\n", 0) \
     M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices", 0) \
@@ -1016,6 +1018,7 @@ class IColumn;
     M(UInt64, input_format_max_bytes_to_read_for_schema_inference, 32 * 1024 * 1024, "The maximum bytes of data to read for automatic schema inference", 0) \
     M(Bool, input_format_csv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in CSV format", 0) \
     M(Bool, input_format_csv_try_infer_numbers_from_strings, false, "Try to infer numbers from string fields while schema inference in CSV format", 0) \
+    M(Bool, input_format_csv_try_infer_strings_from_quoted_tuples, true, "Interpret quoted tuples in the input data as a value of type String.", 0) \
     M(Bool, input_format_tsv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in TSV format", 0) \
     M(Bool, input_format_csv_detect_header, true, "Automatically detect header with names and types in CSV format", 0) \
     M(Bool, input_format_csv_allow_whitespace_or_tab_as_delimiter, false, "Allow to use spaces and tabs(\\t) as field delimiter in the CSV strings", 0) \

src/Core/SettingsChangesHistory.h

@@ -124,6 +124,9 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
               {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
               {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
               {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
+              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
               {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
               }},
     {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},

src/DataTypes/Serializations/SerializationTuple.cpp

@@ -527,26 +527,98 @@ void SerializationTuple::serializeTextXML(const IColumn & column, size_t row_num
 
 void SerializationTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    WriteBufferFromOwnString wb;
-    serializeText(column, row_num, wb, settings);
-    writeCSV(wb.str(), ostr);
+    if (settings.csv.serialize_tuple_into_separate_columns)
+    {
+        for (size_t i = 0; i < elems.size(); ++i)
+        {
+            if (i != 0)
+                writeChar(settings.csv.tuple_delimiter, ostr);
+            elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr, settings);
+        }
+    }
+    else
+    {
+        WriteBufferFromOwnString wb;
+        serializeText(column, row_num, wb, settings);
+        writeCSV(wb.str(), ostr);
+    }
 }
 
 void SerializationTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    String s;
-    readCSV(s, istr, settings.csv);
-    ReadBufferFromString rb(s);
-    deserializeText(column, rb, settings, true);
+    if (settings.csv.deserialize_separate_columns_into_tuple)
+    {
+        addElementSafe<void>(elems.size(), column, [&]
+        {
+            const size_t size = elems.size();
+            for (size_t i = 0; i < size; ++i)
+            {
+                if (i != 0)
+                {
+                    skipWhitespaceIfAny(istr);
+                    assertChar(settings.csv.tuple_delimiter, istr);
+                    skipWhitespaceIfAny(istr);
+                }
+
+                auto & element_column = extractElementColumn(column, i);
+                if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
+                    SerializationNullable::deserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i]);
+                else
+                    elems[i]->deserializeTextCSV(element_column, istr, settings);
+            }
+            return true;
+        });
+    }
+    else
+    {
+        String s;
+        readCSV(s, istr, settings.csv);
+        ReadBufferFromString rb(s);
+        deserializeText(column, rb, settings, true);
+    }
 }
 
 bool SerializationTuple::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    String s;
-    if (!tryReadCSV(s, istr, settings.csv))
-        return false;
-    ReadBufferFromString rb(s);
-    return tryDeserializeText(column, rb, settings, true);
+    if (settings.csv.deserialize_separate_columns_into_tuple)
+    {
+        return addElementSafe<bool>(elems.size(), column, [&]
+        {
+            const size_t size = elems.size();
+            for (size_t i = 0; i < size; ++i)
+            {
+                if (i != 0)
+                {
+                skipWhitespaceIfAny(istr);
+                if (!checkChar(settings.csv.tuple_delimiter, istr))
+                    return false;
+                skipWhitespaceIfAny(istr);
+                }
+
+                auto & element_column = extractElementColumn(column, i);
+                if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
+                {
+                if (!SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i]))
+                    return false;
+                }
+                else
+                {
+                if (!elems[i]->tryDeserializeTextCSV(element_column, istr, settings))
+                    return false;
+                }
+            }
+
+            return true;
+        });
+    }
+    else
+    {
+        String s;
+        if (!tryReadCSV(s, istr, settings.csv))
+            return false;
+        ReadBufferFromString rb(s);
+        return tryDeserializeText(column, rb, settings, true);
+    }
 }
 
 void SerializationTuple::enumerateStreams(

src/Formats/EscapingRuleUtils.cpp

@@ -304,7 +304,7 @@ DataTypePtr tryInferDataTypeByEscapingRule(const String & field, const FormatSet
                 auto type = tryInferDataTypeForSingleField(data, format_settings);
 
                 /// If we couldn't infer any type or it's a number and csv.try_infer_numbers_from_strings = 0, we determine it as a string.
-                if (!type || (isNumber(type) && !format_settings.csv.try_infer_numbers_from_strings))
+                if (!type || (format_settings.csv.try_infer_strings_from_quoted_tuples && isTuple(type)) || (!format_settings.csv.try_infer_numbers_from_strings && isNumber(type)))
                     return std::make_shared<DataTypeString>();
 
                 return type;

src/Formats/FormatFactory.cpp

@@ -76,6 +76,8 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.avro.output_rows_in_file = settings.output_format_avro_rows_in_file;
     format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
     format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
+    format_settings.csv.serialize_tuple_into_separate_columns = settings.output_format_csv_serialize_tuple_into_separate_columns;
+    format_settings.csv.deserialize_separate_columns_into_tuple = settings.input_format_csv_deserialize_separate_columns_into_tuple;
     format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line;
     format_settings.csv.allow_cr_end_of_line = settings.input_format_csv_allow_cr_end_of_line;
     format_settings.csv.delimiter = settings.format_csv_delimiter;
@@ -93,6 +95,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.csv.allow_variable_number_of_columns = settings.input_format_csv_allow_variable_number_of_columns;
     format_settings.csv.use_default_on_bad_values = settings.input_format_csv_use_default_on_bad_values;
     format_settings.csv.try_infer_numbers_from_strings = settings.input_format_csv_try_infer_numbers_from_strings;
+    format_settings.csv.try_infer_strings_from_quoted_tuples = settings.input_format_csv_try_infer_strings_from_quoted_tuples;
     format_settings.hive_text.fields_delimiter = settings.input_format_hive_text_fields_delimiter;
     format_settings.hive_text.collection_items_delimiter = settings.input_format_hive_text_collection_items_delimiter;
     format_settings.hive_text.map_keys_delimiter = settings.input_format_hive_text_map_keys_delimiter;

src/Formats/FormatSettings.h

@@ -152,6 +152,8 @@ struct FormatSettings
         char delimiter = ',';
         bool allow_single_quotes = true;
         bool allow_double_quotes = true;
+        bool serialize_tuple_into_separate_columns = true;
+        bool deserialize_separate_columns_into_tuple = true;
         bool empty_as_default = false;
         bool crlf_end_of_line = false;
         bool allow_cr_end_of_line = false;
@@ -169,6 +171,7 @@ struct FormatSettings
         bool allow_variable_number_of_columns = false;
         bool use_default_on_bad_values = false;
         bool try_infer_numbers_from_strings = true;
+        bool try_infer_strings_from_quoted_tuples = true;
     } csv{};
 
     struct HiveText

tests/queries/0_stateless/00300_csv.reference

@@ -1,11 +1,11 @@
-"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
+"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
 here"
 "x","y","z","a","b"
-"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
+"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
 here"
 "x","y","z","a","b"
 "String","UInt8","Array(UInt8)","Tuple(UInt16, Array(String))","String"
-"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
+"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
 here"
 0,"0","[]","2000-01-01","2000-01-01 00:00:00"
 1,"1","[0]","2000-01-02","2000-01-01 00:00:01"

tests/queries/0_stateless/01016_input_null_as_default.sh

@@ -11,8 +11,8 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE default_by_other_column (a Float32 DEFA
 
 echo 'CSV'
 echo '\N, 1, \N, "2019-07-22", "[10, 20, 30]", \N
-1, world, 3, "2019-07-23", \N, "('\''tuple'\'', 3.14)"
-2, \N, 123, \N, "[]", "('\''test'\'', 2.71828)"
+1, world, 3, "2019-07-23", \N, tuple, 3.14
+2, \N, 123, \N, "[]", test, 2.71828
 3, \N, \N, \N, \N, \N' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT CSV";
 $CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
 $CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";

tests/queries/0_stateless/02246_tsv_csv_best_effort_schema_inference.reference

@@ -97,8 +97,8 @@ c1	Array(Nullable(Bool))
 []
 [NULL]
 [false]
-c1	Tuple(Nullable(Int64), Nullable(Int64), Nullable(Int64))					
-(1,2,3)
+c1	Nullable(String)					
+(1, 2, 3)
 c1	Nullable(String)					
 123.123
 c1	Array(Tuple(Nullable(Int64), Nullable(Int64), Nullable(Int64)))					

tests/queries/0_stateless/02969_auto_format_detection.reference

@@ -82,7 +82,8 @@ CSV
 c1	Nullable(UInt64)					
 c2	Nullable(String)					
 c3	Array(Nullable(UInt64))					
-c4	Tuple(Nullable(UInt64), Nullable(String))					
+c4	Nullable(UInt64)					
+c5	Nullable(String)					
 a	Nullable(String)					
 b	Nullable(String)					
 c	Array(Nullable(String))					

tests/queries/0_stateless/02977_csv_format_support_tuple.sql

@@ -1,5 +1,9 @@
 -- Tags: no-parallel
 
+SET output_format_csv_serialize_tuple_into_separate_columns = false;
+SET input_format_csv_deserialize_separate_columns_into_tuple = false;
+SET input_format_csv_try_infer_strings_from_quoted_tuples = false;
+
 insert into function file('02977_1.csv') select '20240305', 1, ['s', 'd'], map('a', 2), tuple('222', 33, map('abc', 5)) SETTINGS engine_file_truncate_on_insert=1;
 desc file('02977_1.csv');
 select * from file('02977_1.csv') settings max_threads=1;