mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Merge pull request #60994 from bigo-sg/csv-tuple
fix csv format not support tuple
This commit is contained in:
commit
257cdd83d4
@ -7,6 +7,7 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
|
||||
@ -526,68 +527,26 @@ void SerializationTuple::serializeTextXML(const IColumn & column, size_t row_num
|
||||
|
||||
void SerializationTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
for (size_t i = 0; i < elems.size(); ++i)
|
||||
{
|
||||
if (i != 0)
|
||||
writeChar(settings.csv.tuple_delimiter, ostr);
|
||||
elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr, settings);
|
||||
}
|
||||
WriteBufferFromOwnString wb;
|
||||
serializeText(column, row_num, wb, settings);
|
||||
writeCSV(wb.str(), ostr);
|
||||
}
|
||||
|
||||
void SerializationTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
addElementSafe<void>(elems.size(), column, [&]
|
||||
{
|
||||
const size_t size = elems.size();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
if (i != 0)
|
||||
{
|
||||
skipWhitespaceIfAny(istr);
|
||||
assertChar(settings.csv.tuple_delimiter, istr);
|
||||
skipWhitespaceIfAny(istr);
|
||||
}
|
||||
|
||||
auto & element_column = extractElementColumn(column, i);
|
||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
|
||||
SerializationNullable::deserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i]);
|
||||
else
|
||||
elems[i]->deserializeTextCSV(element_column, istr, settings);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
String s;
|
||||
readCSV(s, istr, settings.csv);
|
||||
ReadBufferFromString rb(s);
|
||||
deserializeText(column, rb, settings, true);
|
||||
}
|
||||
|
||||
bool SerializationTuple::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
return addElementSafe<bool>(elems.size(), column, [&]
|
||||
{
|
||||
const size_t size = elems.size();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
if (i != 0)
|
||||
{
|
||||
skipWhitespaceIfAny(istr);
|
||||
if (!checkChar(settings.csv.tuple_delimiter, istr))
|
||||
String s;
|
||||
if (!tryReadCSV(s, istr, settings.csv))
|
||||
return false;
|
||||
skipWhitespaceIfAny(istr);
|
||||
}
|
||||
|
||||
auto & element_column = extractElementColumn(column, i);
|
||||
if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column))
|
||||
{
|
||||
if (!SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i]))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!elems[i]->tryDeserializeTextCSV(element_column, istr, settings))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
ReadBufferFromString rb(s);
|
||||
return tryDeserializeText(column, rb, settings, true);
|
||||
}
|
||||
|
||||
void SerializationTuple::enumerateStreams(
|
||||
|
@ -303,8 +303,8 @@ DataTypePtr tryInferDataTypeByEscapingRule(const String & field, const FormatSet
|
||||
/// Try to determine the type of value inside quotes
|
||||
auto type = tryInferDataTypeForSingleField(data, format_settings);
|
||||
|
||||
/// If we couldn't infer any type or it's tuple in quotes or it's a number and csv.try_infer_numbers_from_strings = 0, we determine it as a string.
|
||||
if (!type || isTuple(type) || (isNumber(type) && !format_settings.csv.try_infer_numbers_from_strings))
|
||||
/// If we couldn't infer any type or it's a number and csv.try_infer_numbers_from_strings = 0, we determine it as a string.
|
||||
if (!type || (isNumber(type) && !format_settings.csv.try_infer_numbers_from_strings))
|
||||
return std::make_shared<DataTypeString>();
|
||||
|
||||
return type;
|
||||
|
@ -1,11 +1,11 @@
|
||||
"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
|
||||
"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
|
||||
here"
|
||||
"x","y","z","a","b"
|
||||
"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
|
||||
"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
|
||||
here"
|
||||
"x","y","z","a","b"
|
||||
"String","UInt8","Array(UInt8)","Tuple(UInt16, Array(String))","String"
|
||||
"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline
|
||||
"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline
|
||||
here"
|
||||
0,"0","[]","2000-01-01","2000-01-01 00:00:00"
|
||||
1,"1","[0]","2000-01-02","2000-01-01 00:00:01"
|
||||
|
Binary file not shown.
@ -1,3 +1,5 @@
|
||||
-- Tags: no-parallel
|
||||
|
||||
SELECT '-- test FORMAT clause --';
|
||||
SET output_format_write_statistics = 0;
|
||||
SELECT number, 'Hello & world' FROM numbers(3) FORMAT Tsv;
|
||||
|
@ -11,8 +11,8 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE default_by_other_column (a Float32 DEFA
|
||||
|
||||
echo 'CSV'
|
||||
echo '\N, 1, \N, "2019-07-22", "[10, 20, 30]", \N
|
||||
1, world, 3, "2019-07-23", \N, tuple, 3.14
|
||||
2, \N, 123, \N, "[]", test, 2.71828
|
||||
1, world, 3, "2019-07-23", \N, "('\''tuple'\'', 3.14)"
|
||||
2, \N, 123, \N, "[]", "('\''test'\'', 2.71828)"
|
||||
3, \N, \N, \N, \N, \N' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT CSV";
|
||||
$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
|
||||
$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";
|
||||
|
@ -97,7 +97,7 @@ c1 Array(Nullable(Bool))
|
||||
[]
|
||||
[NULL]
|
||||
[false]
|
||||
c1 Nullable(String)
|
||||
c1 Tuple(Nullable(Int64), Nullable(Int64), Nullable(Int64))
|
||||
(1,2,3)
|
||||
c1 Nullable(String)
|
||||
123.123
|
||||
|
@ -82,8 +82,7 @@ CSV
|
||||
c1 Nullable(UInt64)
|
||||
c2 Nullable(String)
|
||||
c3 Array(Nullable(UInt64))
|
||||
c4 Nullable(UInt64)
|
||||
c5 Nullable(String)
|
||||
c4 Tuple(Nullable(UInt64), Nullable(String))
|
||||
a Nullable(String)
|
||||
b Nullable(String)
|
||||
c Array(Nullable(String))
|
||||
|
@ -0,0 +1,6 @@
|
||||
c1 Nullable(String)
|
||||
c2 Nullable(Int64)
|
||||
c3 Array(Nullable(String))
|
||||
c4 Map(String, Nullable(Int64))
|
||||
c5 Tuple(Nullable(String), Nullable(Int64), Map(String, Nullable(Int64)))
|
||||
20240305 1 ['s','d'] {'a':2} ('222',33,{'abc':5})
|
@ -0,0 +1,5 @@
|
||||
-- Tags: no-parallel
|
||||
|
||||
insert into function file('02977_1.csv') select '20240305', 1, ['s', 'd'], map('a', 2), tuple('222', 33, map('abc', 5)) SETTINGS engine_file_truncate_on_insert=1;
|
||||
desc file('02977_1.csv');
|
||||
select * from file('02977_1.csv') settings max_threads=1;
|
Loading…
Reference in New Issue
Block a user