This commit is contained in:
Alexander Tokmakov 2019-09-06 22:01:44 +03:00
parent 196e357250
commit d78c82a8ae
5 changed files with 411 additions and 395 deletions

View File

@ -15,6 +15,7 @@
#include <Parsers/CommonParsers.h>
#include <Processors/Formats/Impl/ConstantExpressionTemplate.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Interpreters/convertFieldToType.h>
namespace DB
@ -101,9 +102,14 @@ private:
return false;
if (literal->begin && literal->end)
{
/// Do not replace empty array
if (literal->value.getType() == Field::Types::Array && literal->value.get<Array>().empty())
return false;
/// Do not replace empty array and array of NULLs
if (literal->value.getType() == Field::Types::Array)
{
const Array & array = literal->value.get<Array>();
auto not_null = std::find_if_not(array.begin(), array.end(), [](const auto & elem) { return elem.isNull(); });
if (not_null == array.end())
return true;
}
String column_name = "_dummy_" + std::to_string(replaced_literals.size());
replaced_literals.emplace_back(literal, column_name, force_nullable);
ast = std::make_shared<ASTIdentifier>(column_name);
@ -202,7 +208,7 @@ bool ConstantExpressionTemplate::getDataType(const LiteralInfo & info, DataTypeP
/// It can be Array(Nullable(nested_type))
bool array_of_nullable = false;
if (auto nullable = dynamic_cast<const DataTypeNullable *>(type.get()))
if (auto nullable = dynamic_cast<const DataTypeNullable *>(nested_type.get()))
{
nested_type = nullable->getNestedType();
array_of_nullable = true;
@ -257,11 +263,11 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format
}
skipWhitespaceIfAny(istr);
const IDataType & type = *literals.getByPosition(cur_column).type;
const DataTypePtr & type = literals.getByPosition(cur_column).type;
if (settings.values.accurate_types_of_literals && use_special_parser[cur_column])
parseLiteralAndAssertType(istr, type, cur_column);
parseLiteralAndAssertType(istr, type.get(), cur_column);
else
type.deserializeAsTextQuoted(*columns[cur_column], istr, settings);
type->deserializeAsTextQuoted(*columns[cur_column], istr, settings);
++cur_column;
}
@ -284,23 +290,30 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format
}
}
void ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, const IDataType & type, size_t column_idx)
void ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * complex_type, size_t column_idx)
{
/// TODO faster way to check types without using Parsers
ParserKeyword parser_null("NULL");
ParserNumber parser_number;
ParserArrayOfLiterals parser_array;
const IDataType * type = complex_type;
bool is_array = false;
if (auto array = dynamic_cast<const DataTypeArray *>(type))
{
type = array->getNestedType().get();
is_array = true;
}
bool is_nullable = false;
if (auto nullable = dynamic_cast<const DataTypeNullable *>(type))
{
type = nullable->getNestedType().get();
is_nullable = true;
}
WhichDataType type_info(type);
bool is_array = type_info.isArray();
if (is_array)
type_info = WhichDataType(dynamic_cast<const DataTypeArray &>(type).getNestedType());
bool is_nullable = type_info.isNullable();
if (is_nullable)
type_info = WhichDataType(dynamic_cast<const DataTypeNullable &>(type).getNestedType());
/// If literal does not fit entirely in the buffer, parsing error will happen.
/// However, it's possible to deduce new template after error like it was template mismatch.
/// TODO fix it
@ -327,7 +340,8 @@ void ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, co
(nested_type_info.isNativeInt() && type_info.isInt64()) ||
(nested_type_info.isFloat64() && type_info.isFloat64()))
{
columns[column_idx]->insert(array);
Field array_same_types = convertFieldToType(array, *complex_type, nullptr);
columns[column_idx]->insert(array_same_types);
return;
}
}

View File

@ -33,7 +33,7 @@ public:
private:
static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr);
bool getDataType(const LiteralInfo & info, DataTypePtr & type) const;
void parseLiteralAndAssertType(ReadBuffer & istr, const IDataType & type, size_t column_idx);
void parseLiteralAndAssertType(ReadBuffer & istr, const IDataType * type, size_t column_idx);
private:
DataTypePtr result_column_type;

View File

@ -119,14 +119,12 @@ bool ValuesBlockInputFormat::parseExpressionUsingTemplate(MutableColumnPtr & col
try
{
templates[column_idx].value().parseExpression(buf, format_settings);
assertDelimiterAfterValue(column_idx);
++rows_parsed_using_template[column_idx];
return true;
}
catch (DB::Exception & e)
{
if (e.code() != ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE &&
e.code() != ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED)
if (e.code() != ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE)
throw;
/// Expression in the current row is not match template deduced on the first row.
/// Evaluate expressions, which were parsed using this template.
@ -199,8 +197,14 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
bool parsed = parser.parse(token_iterator, ast, expected);
if (parsed)
{
/// Consider delimiter after value (',' or ')') as part of expression
if (column_idx + 1 != num_columns)
parsed = token_iterator->type == TokenType::Comma;
else
parsed = token_iterator->type == TokenType::ClosingRoundBracket;
++token_iterator;
buf.position() = const_cast<char *>(token_iterator->begin);
parsed = checkDelimiterAfterValue(column_idx);
}
if (!parsed)
{
@ -220,7 +224,6 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
templates[column_idx] = ConstantExpressionTemplate(header.getByPosition(column_idx).type, TokenIterator(tokens), token_iterator, ast, *context);
buf.rollbackToCheckpoint();
templates[column_idx].value().parseExpression(buf, format_settings);
assertDelimiterAfterValue(column_idx);
return;
}
catch (...)
@ -246,7 +249,6 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE};
}
assertDelimiterAfterValue(column_idx);
column.insert(value);
}

View File

@ -1,370 +1,370 @@
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.6542885368159769
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.8444267125384498
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.968375124847465
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.7836319925339997
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.6375535053362572
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5871709781307678
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5202091999924413
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5130525169352177
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.5581075117249046
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.6798311701936688
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7995193250661652
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.7773509726916165
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.8606987912604608
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.635293405011568
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9771089703353683
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.9955717835823895
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6124539775938348
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.6564358792397615
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.5521115589991579
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.7792659923782862
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.6656871036437929
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.743513774337199
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.8688023472919778
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.7225690042828818
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8866100282141612
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8374461350184258
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.8365104788783658
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.928892180915439
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.7275019293899533
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.9516437185963472
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.654288536816
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.844426712538
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.968375124847
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.783631992534
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.637553505336
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.587170978131
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.520209199992
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.513052516935
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.558107511725
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.679831170194
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.799519325066
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.777350972692
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.86069879126
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.635293405012
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.977108970335
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.995571783582
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.612453977594
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.65643587924
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.552111558999
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.779265992378
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.665687103644
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.743513774337
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.868802347292
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.722569004283
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.886610028214
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.837446135018
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.836510478878
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.928892180915
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.72750192939
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596
0.951643718596

File diff suppressed because one or more lines are too long