thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 09:10:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

setup csv delimiter in Template format

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-10-07 19:03:01 +03:00
parent b4d9dc2a90
commit 32b0b8272c
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
View File

@ -24,7 +24,8 @@ TemplateRowInputFormat::TemplateRowInputFormat(const Block & header_, ReadBuffer
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_) ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
: RowInputFormatWithDiagnosticInfo(header_, buf, params_), buf(in_), data_types(header_.getDataTypes()), : RowInputFormatWithDiagnosticInfo(header_, buf, params_), buf(in_), data_types(header_.getDataTypes()),
settings(settings_), ignore_spaces(ignore_spaces_), settings(settings_), ignore_spaces(ignore_spaces_),
format(std::move(format_)), row_format(std::move(row_format_)) format(std::move(format_)), row_format(std::move(row_format_)),
default_csv_delimiter(settings.csv.delimiter)
{ {
/// Validate format string for result set /// Validate format string for result set
bool has_data = false; bool has_data = false;
@ -167,7 +168,7 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
if (row_format.format_idx_to_column_idx[i]) if (row_format.format_idx_to_column_idx[i])
{ {
size_t col_idx = *row_format.format_idx_to_column_idx[i]; size_t col_idx = *row_format.format_idx_to_column_idx[i];
extra.read_columns[col_idx] = deserializeField(data_types[col_idx], *columns[col_idx], row_format.formats[i]); extra.read_columns[col_idx] = deserializeField(data_types[col_idx], *columns[col_idx], i);
} }
else else
skipField(row_format.formats[i]); skipField(row_format.formats[i]);
@ -184,8 +185,9 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
return true; return true;
} }
bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn & column, ColumnFormat col_format) bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn & column, size_t file_column)
{ {
ColumnFormat col_format = row_format.formats[file_column];
bool read = true; bool read = true;
bool parse_as_nullable = settings.null_as_default && !type->isNullable(); bool parse_as_nullable = settings.null_as_default && !type->isNullable();
try try
@ -205,6 +207,9 @@ bool TemplateRowInputFormat::deserializeField(const DataTypePtr & type, IColumn
type->deserializeAsTextQuoted(column, buf, settings); type->deserializeAsTextQuoted(column, buf, settings);
break; break;
case ColumnFormat::Csv: case ColumnFormat::Csv:
/// Will read unquoted string until settings.csv.delimiter
settings.csv.delimiter = row_format.delimiters[file_column + 1].empty() ? default_csv_delimiter :
row_format.delimiters[file_column + 1].front();
if (parse_as_nullable) if (parse_as_nullable)
read = DataTypeNullable::deserializeTextCSV(column, buf, settings, type); read = DataTypeNullable::deserializeTextCSV(column, buf, settings, type);
else else
@ -406,7 +411,7 @@ void TemplateRowInputFormat::tryDeserializeFiled(const DataTypePtr & type, IColu
{ {
prev_pos = buf.position(); prev_pos = buf.position();
if (row_format.format_idx_to_column_idx[file_column]) if (row_format.format_idx_to_column_idx[file_column])
deserializeField(type, column, row_format.formats[file_column]); deserializeField(type, column, file_column);
else else
skipField(row_format.formats[file_column]); skipField(row_format.formats[file_column]);
curr_pos = buf.position(); curr_pos = buf.position();

5
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.h
View File

@ -29,7 +29,7 @@ public:
void syncAfterError() override; void syncAfterError() override;
private: private:
bool deserializeField(const DataTypePtr & type, IColumn & column, ColumnFormat col_format); bool deserializeField(const DataTypePtr & type, IColumn & column, size_t file_column);
void skipField(ColumnFormat col_format); void skipField(ColumnFormat col_format);
inline void skipSpaces() { if (ignore_spaces) skipWhitespaceIfAny(buf); } inline void skipSpaces() { if (ignore_spaces) skipWhitespaceIfAny(buf); }
@ -50,13 +50,14 @@ private:
PeekableReadBuffer buf; PeekableReadBuffer buf;
DataTypes data_types; DataTypes data_types;
const FormatSettings settings; FormatSettings settings;
const bool ignore_spaces; const bool ignore_spaces;
ParsedTemplateFormatString format; ParsedTemplateFormatString format;
ParsedTemplateFormatString row_format; ParsedTemplateFormatString row_format;
size_t format_data_idx; size_t format_data_idx;
bool end_of_stream = false; bool end_of_stream = false;
char default_csv_delimiter;
}; };
} }