thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 01:00:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

setup csv delimiter in Template format

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-10-07 19:03:01 +03:00
parent b4d9dc2a90
commit 32b0b8272c
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
View File

@ -24,7 +24,8 @@ TemplateRowInputFormat::TemplateRowInputFormat(const Block & header_, ReadBuffer
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
: RowInputFormatWithDiagnosticInfo(header_, buf, params_), buf(in_), data_types(header_.getDataTypes()),
settings(settings_), ignore_spaces(ignore_spaces_),
format(std::move(format_)), row_format(std::move(row_format_))
format(std::move(format_)), row_format(std::move(row_format_)),
default_csv_delimiter(settings.csv.delimiter)
{
/// Validate format string for result set
bool has_data = false;
@ -167,7 +168,7 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
if (row_format.format_idx_to_column_idx[i])
{
size_t col_idx = *row_format.format_idx_to_column_idx[i];
extra.read_columns[col_idx] = deserializeField(data_types[col_idx], *columns[col_idx], row_format.formats[i]);
extra.read_columns[col_idx] = deserializeField(data_types[col_idx], *columns[col_idx], i);
}
else
skipField(row_format.formats[i]);
@ -184,8 +185,9 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
return true;
}
bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn & column, ColumnFormat col_format)
bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn & column, size_t file_column)
{
ColumnFormat col_format = row_format.formats[file_column];
bool read = true;
bool parse_as_nullable = settings.null_as_default && !type->isNullable();
try
@ -205,6 +207,9 @@ bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn
type->deserializeAsTextQuoted(column, buf, settings);
break;
case ColumnFormat::Csv:
/// Will read unquoted string until settings.csv.delimiter
settings.csv.delimiter = row_format.delimiters[file_column + 1].empty() ? default_csv_delimiter :
row_format.delimiters[file_column + 1].front();
if (parse_as_nullable)
read = DataTypeNullable::deserializeTextCSV(column, buf, settings, type);
else
@ -406,7 +411,7 @@ void TemplateRowInputFormat::tryDeserializeFiled(const DataTypePtr & type, IColu
{
prev_pos = buf.position();
if (row_format.format_idx_to_column_idx[file_column])
deserializeField(type, column, row_format.formats[file_column]);
deserializeField(type, column, file_column);
else
skipField(row_format.formats[file_column]);
curr_pos = buf.position();

5
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.h
View File

@ -29,7 +29,7 @@ public:
void syncAfterError() override;
private:
bool deserializeField(const DataTypePtr & type, IColumn & column, ColumnFormat col_format);
bool deserializeField(const DataTypePtr & type, IColumn & column, size_t file_column);
void skipField(ColumnFormat col_format);
inline void skipSpaces() { if (ignore_spaces) skipWhitespaceIfAny(buf); }
@ -50,13 +50,14 @@ private:
PeekableReadBuffer buf;
DataTypes data_types;
const FormatSettings settings;
FormatSettings settings;
const bool ignore_spaces;
ParsedTemplateFormatString format;
ParsedTemplateFormatString row_format;
size_t format_data_idx;
bool end_of_stream = false;
char default_csv_delimiter;
};
}