thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 13:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert parsing expression near the end of buffer

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-09-05 16:55:51 +03:00
parent 20770a9745
commit 196e357250
1 changed files with 14 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
View File

@ -86,11 +86,6 @@ Chunk ValuesBlockInputFormat::generate()
throw; throw;
} }
} }
if (!buf.eof())
{
assertChar(';', buf);
skipWhitespaceIfAny(buf);
}
/// Evaluate expressions, which were parsed using templates, if any /// Evaluate expressions, which were parsed using templates, if any
for (size_t i = 0; i < columns.size(); ++i) for (size_t i = 0; i < columns.size(); ++i)
@ -196,34 +191,23 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
const IDataType & type = *header.getByPosition(column_idx).type; const IDataType & type = *header.getByPosition(column_idx).type;
Expected expected; Expected expected;
auto tokens = std::make_unique<Tokens>(buf.position(), buf.buffer().end()); /// TODO use FileSegmentationEngineValues from #6553 to find end of expression and get continuous memory containing expression
IParser::Pos token_iterator(*tokens); Tokens tokens(buf.position(), buf.buffer().end());
IParser::Pos token_iterator(tokens);
ASTPtr ast; ASTPtr ast;
bool near_the_end_of_buffer = true; bool parsed = parser.parse(token_iterator, ast, expected);
bool can_peek = true; if (parsed)
while (near_the_end_of_buffer)
{ {
bool parsed = parser.parse(token_iterator, ast, expected);
buf.position() = const_cast<char *>(token_iterator->begin); buf.position() = const_cast<char *>(token_iterator->begin);
parsed &= checkDelimiterAfterValue(column_idx); parsed = checkDelimiterAfterValue(column_idx);
}
near_the_end_of_buffer = buf.available() < 256 && can_peek; if (!parsed)
if (near_the_end_of_buffer) {
{ buf.rollbackToCheckpoint();
/// Rare case: possibly only some prefix of expression was parsed. Peek more data and try again throw Exception("Cannot parse expression of type " + type.getName() + " here: "
can_peek = buf.peekNext(); + String(buf.position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf.buffer().end() - buf.position())),
buf.rollbackToCheckpoint(); ErrorCodes::SYNTAX_ERROR);
tokens = std::make_unique<Tokens>(buf.position(), buf.buffer().end());
token_iterator = IParser::Pos{*tokens};
}
else if (!parsed)
{
buf.rollbackToCheckpoint();
throw Exception("Cannot parse expression of type " + type.getName() + " here: "
+ String(buf.position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf.buffer().end() - buf.position())),
ErrorCodes::SYNTAX_ERROR);
}
} }
if (format_settings.values.deduce_templates_of_expressions && deduce_template) if (format_settings.values.deduce_templates_of_expressions && deduce_template)
@ -233,7 +217,7 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
ErrorCodes::LOGICAL_ERROR); ErrorCodes::LOGICAL_ERROR);
try try
{ {
templates[column_idx] = ConstantExpressionTemplate(header.getByPosition(column_idx).type, TokenIterator(*tokens), token_iterator, ast, *context); templates[column_idx] = ConstantExpressionTemplate(header.getByPosition(column_idx).type, TokenIterator(tokens), token_iterator, ast, *context);
buf.rollbackToCheckpoint(); buf.rollbackToCheckpoint();
templates[column_idx].value().parseExpression(buf, format_settings); templates[column_idx].value().parseExpression(buf, format_settings);
assertDelimiterAfterValue(column_idx); assertDelimiterAfterValue(column_idx);