mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
better checking of types of literals
This commit is contained in:
parent
b9eb992320
commit
8936f854be
@ -449,9 +449,7 @@ namespace ErrorCodes
|
|||||||
extern const int READONLY_SETTING = 472;
|
extern const int READONLY_SETTING = 472;
|
||||||
extern const int DEADLOCK_AVOIDED = 473;
|
extern const int DEADLOCK_AVOIDED = 473;
|
||||||
extern const int INVALID_TEMPLATE_FORMAT = 474;
|
extern const int INVALID_TEMPLATE_FORMAT = 474;
|
||||||
extern const int CANNOT_CREATE_EXPRESSION_TEMPLATE = 475;
|
|
||||||
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE = 476;
|
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE = 476;
|
||||||
extern const int CANNOT_EVALUATE_EXPRESSION_TEMPLATE = 477;
|
|
||||||
|
|
||||||
extern const int KEEPER_EXCEPTION = 999;
|
extern const int KEEPER_EXCEPTION = 999;
|
||||||
extern const int POCO_EXCEPTION = 1000;
|
extern const int POCO_EXCEPTION = 1000;
|
||||||
|
@ -501,7 +501,6 @@ public:
|
|||||||
bool isVariadic() const override { return function->isVariadic(); }
|
bool isVariadic() const override { return function->isVariadic(); }
|
||||||
size_t getNumberOfArguments() const override { return function->getNumberOfArguments(); }
|
size_t getNumberOfArguments() const override { return function->getNumberOfArguments(); }
|
||||||
|
|
||||||
// FIXME it's a temporary hack for ConstantExpressionTemplate
|
|
||||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return function->getArgumentsThatAreAlwaysConstant(); }
|
ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return function->getArgumentsThatAreAlwaysConstant(); }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
|
|
||||||
#include <Columns/ColumnConst.h>
|
#include <Columns/ColumnConst.h>
|
||||||
#include <DataTypes/DataTypesNumber.h>
|
#include <DataTypes/DataTypesNumber.h>
|
||||||
#include <DataTypes/DataTypeNullable.h>
|
#include <DataTypes/DataTypeNullable.h>
|
||||||
@ -23,52 +22,53 @@ namespace DB
|
|||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int CANNOT_CREATE_EXPRESSION_TEMPLATE;
|
|
||||||
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE;
|
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE;
|
||||||
extern const int CANNOT_EVALUATE_EXPRESSION_TEMPLATE;
|
|
||||||
extern const int SYNTAX_ERROR;
|
extern const int SYNTAX_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct LiteralInfo
|
||||||
|
{
|
||||||
|
typedef std::shared_ptr<ASTLiteral> ASTLiteralPtr;
|
||||||
|
LiteralInfo(const ASTLiteralPtr & literal_, const String & column_name_, bool force_nullable_)
|
||||||
|
: literal(literal_), dummy_column_name(column_name_), force_nullable(force_nullable_) { }
|
||||||
|
ASTLiteralPtr literal;
|
||||||
|
String dummy_column_name;
|
||||||
|
/// Make column nullable even if expression type is not.
|
||||||
|
/// (for literals in functions like ifNull and assumeNotNul, which never return NULL even for NULL arguments)
|
||||||
|
bool force_nullable;
|
||||||
|
};
|
||||||
|
|
||||||
|
using LiteralsInfo = std::vector<LiteralInfo>;
|
||||||
|
|
||||||
class ReplaceLiteralsVisitor
|
class ReplaceLiteralsVisitor
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
|
|
||||||
struct LiteralInfo
|
|
||||||
{
|
|
||||||
typedef std::shared_ptr<ASTLiteral> ASTLiteralPtr;
|
|
||||||
LiteralInfo(const ASTLiteralPtr & literal_, const String & column_name_) : literal(literal_), dummy_column_name(column_name_) { }
|
|
||||||
ASTLiteralPtr literal;
|
|
||||||
String dummy_column_name;
|
|
||||||
};
|
|
||||||
|
|
||||||
using LiteralsInfo = std::vector<LiteralInfo>;
|
|
||||||
|
|
||||||
LiteralsInfo replaced_literals;
|
LiteralsInfo replaced_literals;
|
||||||
const Context & context;
|
const Context & context;
|
||||||
|
|
||||||
explicit ReplaceLiteralsVisitor(const Context & context_) : context(context_) { }
|
explicit ReplaceLiteralsVisitor(const Context & context_) : context(context_) { }
|
||||||
|
|
||||||
void visit(ASTPtr & ast)
|
void visit(ASTPtr & ast, bool force_nullable = {})
|
||||||
{
|
{
|
||||||
if (visitIfLiteral(ast))
|
if (visitIfLiteral(ast, force_nullable))
|
||||||
return;
|
return;
|
||||||
if (auto function = ast->as<ASTFunction>())
|
if (auto function = ast->as<ASTFunction>())
|
||||||
visit(*function);
|
visit(*function, force_nullable);
|
||||||
else if (ast->as<ASTLiteral>())
|
else if (ast->as<ASTLiteral>())
|
||||||
throw DB::Exception("Identifier in constant expression", ErrorCodes::SYNTAX_ERROR);
|
throw DB::Exception("Identifier in constant expression", ErrorCodes::SYNTAX_ERROR);
|
||||||
else
|
else
|
||||||
visitChildren(ast, {});
|
visitChildren(ast, {}, std::vector<char>(ast->children.size(), force_nullable));
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void visitChildren(ASTPtr & ast, const ColumnNumbers & dont_visit_children)
|
void visitChildren(ASTPtr & ast, const ColumnNumbers & dont_visit_children, const std::vector<char> & force_nullable)
|
||||||
{
|
{
|
||||||
for (size_t i = 0; i < ast->children.size(); ++i)
|
for (size_t i = 0; i < ast->children.size(); ++i)
|
||||||
if (std::find(dont_visit_children.begin(), dont_visit_children.end(), i) == dont_visit_children.end())
|
if (std::find(dont_visit_children.begin(), dont_visit_children.end(), i) == dont_visit_children.end())
|
||||||
visit(ast->children[i]);
|
visit(ast->children[i], force_nullable[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
void visit(ASTFunction & function)
|
void visit(ASTFunction & function, bool force_nullable)
|
||||||
{
|
{
|
||||||
/// Do not replace literals which must be constant
|
/// Do not replace literals which must be constant
|
||||||
ColumnNumbers dont_visit_children;
|
ColumnNumbers dont_visit_children;
|
||||||
@ -78,53 +78,70 @@ private:
|
|||||||
dont_visit_children = default_builder->getArgumentsThatAreAlwaysConstant();
|
dont_visit_children = default_builder->getArgumentsThatAreAlwaysConstant();
|
||||||
else if (dynamic_cast<FunctionBuilderCast*>(builder.get()))
|
else if (dynamic_cast<FunctionBuilderCast*>(builder.get()))
|
||||||
dont_visit_children.push_back(1);
|
dont_visit_children.push_back(1);
|
||||||
|
/// FIXME suppose there is no other functions, which require constant arguments (it's true, until the new one is added)
|
||||||
|
|
||||||
visitChildren(function.arguments, dont_visit_children);
|
/// Allow nullable arguments if function never returns NULL
|
||||||
|
bool return_not_null = function.name == "isNull" || function.name == "isNotNull" ||
|
||||||
|
function.name == "ifNull" || function.name == "assumeNotNull" ||
|
||||||
|
function.name == "coalesce";
|
||||||
|
|
||||||
|
std::vector<char> force_nullable_arguments(function.arguments->children.size(), force_nullable || return_not_null);
|
||||||
|
|
||||||
|
/// coalesce may return NULL if the last argument is NULL
|
||||||
|
if (!force_nullable && function.name == "coalesce")
|
||||||
|
force_nullable_arguments.back() = false;
|
||||||
|
|
||||||
|
visitChildren(function.arguments, dont_visit_children, force_nullable_arguments);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool visitIfLiteral(ASTPtr & ast)
|
bool visitIfLiteral(ASTPtr & ast, bool force_nullable)
|
||||||
{
|
{
|
||||||
auto literal = std::dynamic_pointer_cast<ASTLiteral>(ast);
|
auto literal = std::dynamic_pointer_cast<ASTLiteral>(ast);
|
||||||
if (!literal)
|
if (!literal)
|
||||||
return false;
|
return false;
|
||||||
if (literal->begin && literal->end)
|
if (literal->begin && literal->end)
|
||||||
{
|
{
|
||||||
|
/// Do not replace empty array
|
||||||
|
if (literal->value.getType() == Field::Types::Array && literal->value.get<Array>().empty())
|
||||||
|
return false;
|
||||||
String column_name = "_dummy_" + std::to_string(replaced_literals.size());
|
String column_name = "_dummy_" + std::to_string(replaced_literals.size());
|
||||||
replaced_literals.emplace_back(literal, column_name);
|
replaced_literals.emplace_back(literal, column_name, force_nullable);
|
||||||
ast = std::make_shared<ASTIdentifier>(column_name);
|
ast = std::make_shared<ASTIdentifier>(column_name);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
using LiteralInfo = ReplaceLiteralsVisitor::LiteralInfo;
|
|
||||||
using LiteralsInfo = ReplaceLiteralsVisitor::LiteralsInfo;
|
|
||||||
|
|
||||||
|
/// Expression template is a sequence of tokens and data types of literals.
|
||||||
ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_column_type,
|
/// E.g. template of "position('some string', 'other string') != 0" is
|
||||||
|
/// ["position", "(", DataTypeString, ",", DataTypeString, ")", "!=", DataTypeUInt64]
|
||||||
|
ConstantExpressionTemplate::ConstantExpressionTemplate(DataTypePtr result_column_type_,
|
||||||
TokenIterator expression_begin, TokenIterator expression_end,
|
TokenIterator expression_begin, TokenIterator expression_end,
|
||||||
const ASTPtr & expression_, const Context & context)
|
const ASTPtr & expression_, const Context & context)
|
||||||
|
: result_column_type(result_column_type_)
|
||||||
{
|
{
|
||||||
|
/// Extract ASTLiterals from expression and replace them with ASTIdentifiers where needed
|
||||||
ASTPtr expression = expression_->clone();
|
ASTPtr expression = expression_->clone();
|
||||||
ReplaceLiteralsVisitor visitor(context);
|
ReplaceLiteralsVisitor visitor(context);
|
||||||
visitor.visit(expression);
|
visitor.visit(expression);
|
||||||
LiteralsInfo replaced_literals = visitor.replaced_literals;
|
LiteralsInfo & replaced_literals = visitor.replaced_literals;
|
||||||
|
|
||||||
token_after_literal_idx.reserve(replaced_literals.size());
|
|
||||||
need_special_parser.resize(replaced_literals.size(), true);
|
|
||||||
|
|
||||||
std::sort(replaced_literals.begin(), replaced_literals.end(), [](const LiteralInfo & a, const LiteralInfo & b)
|
std::sort(replaced_literals.begin(), replaced_literals.end(), [](const LiteralInfo & a, const LiteralInfo & b)
|
||||||
{
|
{
|
||||||
return a.literal->begin.value() < b.literal->begin.value();
|
return a.literal->begin.value() < b.literal->begin.value();
|
||||||
});
|
});
|
||||||
|
|
||||||
bool allow_nulls = result_column_type.isNullable();
|
/// Make sequence of tokens and determine IDataType by Field::Types:Which for each literal.
|
||||||
|
token_after_literal_idx.reserve(replaced_literals.size());
|
||||||
|
use_special_parser.resize(replaced_literals.size(), true);
|
||||||
|
|
||||||
TokenIterator prev_end = expression_begin;
|
TokenIterator prev_end = expression_begin;
|
||||||
for (size_t i = 0; i < replaced_literals.size(); ++i)
|
for (size_t i = 0; i < replaced_literals.size(); ++i)
|
||||||
{
|
{
|
||||||
const LiteralInfo & info = replaced_literals[i];
|
const LiteralInfo & info = replaced_literals[i];
|
||||||
if (info.literal->begin.value() < prev_end)
|
if (info.literal->begin.value() < prev_end)
|
||||||
throw Exception("Cannot replace literals", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
|
throw Exception("Cannot replace literals", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
while (prev_end < info.literal->begin.value())
|
while (prev_end < info.literal->begin.value())
|
||||||
{
|
{
|
||||||
@ -133,21 +150,8 @@ ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_
|
|||||||
}
|
}
|
||||||
token_after_literal_idx.push_back(tokens.size());
|
token_after_literal_idx.push_back(tokens.size());
|
||||||
|
|
||||||
DataTypePtr type = applyVisitor(FieldToDataType(), info.literal->value);
|
DataTypePtr type;
|
||||||
|
use_special_parser[i] = getDataType(info, type);
|
||||||
WhichDataType type_info(type);
|
|
||||||
if (type_info.isNativeInt())
|
|
||||||
type = std::make_shared<DataTypeInt64>();
|
|
||||||
else if (type_info.isNativeUInt())
|
|
||||||
type = std::make_shared<DataTypeUInt64>();
|
|
||||||
else if (!type_info.isFloat())
|
|
||||||
need_special_parser[i] = false;
|
|
||||||
|
|
||||||
/// Allow literal to be NULL, if result column has nullable type
|
|
||||||
// TODO also allow NULL literals inside functions, which return not NULL for NULL arguments,
|
|
||||||
// even if result_column_type is not nullable
|
|
||||||
if (allow_nulls && type->canBeInsideNullable())
|
|
||||||
type = makeNullable(type);
|
|
||||||
|
|
||||||
literals.insert({nullptr, type, info.dummy_column_name});
|
literals.insert({nullptr, type, info.dummy_column_name});
|
||||||
|
|
||||||
@ -161,68 +165,102 @@ ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_
|
|||||||
}
|
}
|
||||||
|
|
||||||
columns = literals.cloneEmptyColumns();
|
columns = literals.cloneEmptyColumns();
|
||||||
addNodesToCastResult(result_column_type, expression);
|
addNodesToCastResult(*result_column_type, expression);
|
||||||
result_column_name = expression->getColumnName();
|
result_column_name = expression->getColumnName();
|
||||||
|
|
||||||
// TODO convert SyntaxAnalyzer and ExpressionAnalyzer exceptions to CANNOT_CREATE_EXPRESSION_TEMPLATE
|
|
||||||
auto syntax_result = SyntaxAnalyzer(context).analyze(expression, literals.getNamesAndTypesList());
|
auto syntax_result = SyntaxAnalyzer(context).analyze(expression, literals.getNamesAndTypesList());
|
||||||
|
|
||||||
actions_on_literals = ExpressionAnalyzer(expression, syntax_result, context).getActions(false);
|
actions_on_literals = ExpressionAnalyzer(expression, syntax_result, context).getActions(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool ConstantExpressionTemplate::getDataType(const LiteralInfo & info, DataTypePtr & type) const
|
||||||
|
{
|
||||||
|
/// Type (Field::Types:Which) of literal in AST can be: String, UInt64, Int64, Float64, Null or Array of simple literals (not of Arrays).
|
||||||
|
/// Null and empty Array literals are considered as tokens, because template with Nullable<Nothing> or Array<Nothing> is useless.
|
||||||
|
|
||||||
|
Field::Types::Which field_type = info.literal->value.getType();
|
||||||
|
|
||||||
|
/// We have to use ParserNumber instead of type->deserializeAsTextQuoted() for arithmetic types
|
||||||
|
/// to check actual type of literal and avoid possible overflow and precision issues.
|
||||||
|
bool need_special_parser = true;
|
||||||
|
|
||||||
|
/// Do not use 8, 16 and 32 bit types, so template will match all integers
|
||||||
|
if (field_type == Field::Types::UInt64)
|
||||||
|
type = std::make_shared<DataTypeUInt64>();
|
||||||
|
else if (field_type == Field::Types::Int64)
|
||||||
|
type = std::make_shared<DataTypeInt64>();
|
||||||
|
else if (field_type == Field::Types::Float64)
|
||||||
|
type = std::make_shared<DataTypeFloat64>();
|
||||||
|
else if (field_type == Field::Types::String)
|
||||||
|
{
|
||||||
|
need_special_parser = false;
|
||||||
|
type = std::make_shared<DataTypeString>();
|
||||||
|
}
|
||||||
|
else if (field_type == Field::Types::Array)
|
||||||
|
{
|
||||||
|
type = applyVisitor(FieldToDataType(), info.literal->value);
|
||||||
|
auto nested_type = dynamic_cast<const DataTypeArray &>(*type).getNestedType();
|
||||||
|
|
||||||
|
/// It can be Array<Nullable<nested_type>>
|
||||||
|
bool array_of_nullable = false;
|
||||||
|
if (auto nullable = dynamic_cast<const DataTypeNullable *>(type.get()))
|
||||||
|
{
|
||||||
|
nested_type = nullable->getNestedType();
|
||||||
|
array_of_nullable = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
WhichDataType type_info{nested_type};
|
||||||
|
/// Promote integers to 64 bit types
|
||||||
|
if (type_info.isNativeUInt())
|
||||||
|
nested_type = std::make_shared<DataTypeUInt64>();
|
||||||
|
else if (type_info.isNativeInt())
|
||||||
|
nested_type = std::make_shared<DataTypeInt64>();
|
||||||
|
else if (type_info.isFloat64())
|
||||||
|
;
|
||||||
|
else if (type_info.isString())
|
||||||
|
need_special_parser = false;
|
||||||
|
else
|
||||||
|
throw Exception("Unexpected literal type inside Array: " + nested_type->getName() + ". It's a bug",
|
||||||
|
ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
|
if (array_of_nullable)
|
||||||
|
nested_type = std::make_shared<DataTypeNullable>(nested_type);
|
||||||
|
|
||||||
|
type = std::make_shared<DataTypeArray>(nested_type);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
throw Exception(String("Unexpected literal type ") + info.literal->value.getTypeName() + ". It's a bug",
|
||||||
|
ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
|
/// Allow literal to be NULL, if result column has nullable type or if function never returns NULL
|
||||||
|
bool allow_nulls = result_column_type->isNullable();
|
||||||
|
if ((allow_nulls || info.force_nullable) && type->canBeInsideNullable())
|
||||||
|
type = makeNullable(type);
|
||||||
|
|
||||||
|
return need_special_parser;
|
||||||
|
}
|
||||||
|
|
||||||
void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const FormatSettings & settings)
|
void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const FormatSettings & settings)
|
||||||
{
|
{
|
||||||
size_t cur_column = 0;
|
size_t cur_column = 0;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
ParserKeyword parser_null("NULL");
|
|
||||||
ParserNumber parser_number;
|
|
||||||
size_t cur_token = 0;
|
size_t cur_token = 0;
|
||||||
while (cur_column < literals.columns())
|
while (cur_column < literals.columns())
|
||||||
{
|
{
|
||||||
size_t skip_tokens_until = token_after_literal_idx[cur_column];
|
size_t skip_tokens_until = token_after_literal_idx[cur_column];
|
||||||
while (cur_token < skip_tokens_until)
|
while (cur_token < skip_tokens_until)
|
||||||
{
|
{
|
||||||
// TODO skip comments
|
/// TODO skip comments
|
||||||
skipWhitespaceIfAny(istr);
|
skipWhitespaceIfAny(istr);
|
||||||
assertString(tokens[cur_token++], istr);
|
assertString(tokens[cur_token++], istr);
|
||||||
}
|
}
|
||||||
skipWhitespaceIfAny(istr);
|
skipWhitespaceIfAny(istr);
|
||||||
|
|
||||||
const IDataType & type = *literals.getByPosition(cur_column).type;
|
const IDataType & type = *literals.getByPosition(cur_column).type;
|
||||||
if (need_special_parser[cur_column])
|
if (use_special_parser[cur_column])
|
||||||
{
|
parseLiteralAndAssertType(istr, type, cur_column);
|
||||||
WhichDataType type_info(type);
|
|
||||||
bool nullable = type_info.isNullable();
|
|
||||||
if (nullable)
|
|
||||||
type_info = WhichDataType(dynamic_cast<const DataTypeNullable &>(type).getNestedType());
|
|
||||||
|
|
||||||
Tokens tokens_number(istr.position(), istr.buffer().end());
|
|
||||||
IParser::Pos iterator(tokens_number);
|
|
||||||
Expected expected;
|
|
||||||
ASTPtr ast;
|
|
||||||
if (nullable && parser_null.parse(iterator, ast, expected))
|
|
||||||
ast = std::make_shared<ASTLiteral>(Field());
|
|
||||||
else if (!parser_number.parse(iterator, ast, expected))
|
|
||||||
throw DB::Exception("Cannot parse literal", ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE);
|
|
||||||
istr.position() = const_cast<char *>(iterator->begin);
|
|
||||||
Field & number = ast->as<ASTLiteral&>().value;
|
|
||||||
|
|
||||||
// TODO also check type of Array(T), if T is arithmetic
|
|
||||||
if ((number.getType() == Field::Types::UInt64 && type_info.isUInt64())
|
|
||||||
|| (number.getType() == Field::Types::Int64 && type_info.isInt64())
|
|
||||||
|| (number.getType() == Field::Types::Float64 && type_info.isFloat64())
|
|
||||||
|| nullable)
|
|
||||||
{
|
|
||||||
columns[cur_column]->insert(number);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
throw DB::Exception("Cannot parse literal", ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE);
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
|
||||||
type.deserializeAsTextQuoted(*columns[cur_column], istr, settings);
|
type.deserializeAsTextQuoted(*columns[cur_column], istr, settings);
|
||||||
}
|
|
||||||
|
|
||||||
++cur_column;
|
++cur_column;
|
||||||
}
|
}
|
||||||
@ -245,6 +283,75 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, const IDataType & type, size_t column_idx)
|
||||||
|
{
|
||||||
|
/// TODO faster way to check types without using Parsers
|
||||||
|
ParserKeyword parser_null("NULL");
|
||||||
|
ParserNumber parser_number;
|
||||||
|
ParserArrayOfLiterals parser_array;
|
||||||
|
|
||||||
|
WhichDataType type_info(type);
|
||||||
|
|
||||||
|
bool is_array = type_info.isArray();
|
||||||
|
if (is_array)
|
||||||
|
type_info = WhichDataType(dynamic_cast<const DataTypeArray &>(type).getNestedType());
|
||||||
|
|
||||||
|
bool is_nullable = type_info.isNullable();
|
||||||
|
if (is_nullable)
|
||||||
|
type_info = WhichDataType(dynamic_cast<const DataTypeNullable &>(type).getNestedType());
|
||||||
|
|
||||||
|
/// If literal does not fit entirely in the buffer, parsing error will happen.
|
||||||
|
/// However, it's possible to deduce new template after error like it was template mismatch.
|
||||||
|
/// TODO fix it
|
||||||
|
Tokens tokens_number(istr.position(), istr.buffer().end());
|
||||||
|
IParser::Pos iterator(tokens_number);
|
||||||
|
Expected expected;
|
||||||
|
ASTPtr ast;
|
||||||
|
|
||||||
|
if (is_array)
|
||||||
|
{
|
||||||
|
if (!parser_array.parse(iterator, ast, expected))
|
||||||
|
throw DB::Exception("Cannot parse literal", ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE);
|
||||||
|
istr.position() = const_cast<char *>(iterator->begin);
|
||||||
|
|
||||||
|
const Field & array = ast->as<ASTLiteral &>().value;
|
||||||
|
auto array_type = applyVisitor(FieldToDataType(), array);
|
||||||
|
auto nested_type = dynamic_cast<const DataTypeArray &>(*array_type).getNestedType();
|
||||||
|
if (is_nullable)
|
||||||
|
if (auto nullable = dynamic_cast<const DataTypeNullable *>(nested_type.get()))
|
||||||
|
nested_type = nullable->getNestedType();
|
||||||
|
|
||||||
|
WhichDataType nested_type_info(nested_type);
|
||||||
|
if ((nested_type_info.isNativeUInt() && type_info.isUInt64()) ||
|
||||||
|
(nested_type_info.isNativeInt() && type_info.isInt64()) ||
|
||||||
|
(nested_type_info.isFloat64() && type_info.isFloat64()))
|
||||||
|
{
|
||||||
|
columns[column_idx]->insert(array);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (is_nullable && parser_null.parse(iterator, ast, expected))
|
||||||
|
ast = std::make_shared<ASTLiteral>(Field());
|
||||||
|
else if (!parser_number.parse(iterator, ast, expected))
|
||||||
|
throw DB::Exception("Cannot parse literal", ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE);
|
||||||
|
istr.position() = const_cast<char *>(iterator->begin);
|
||||||
|
|
||||||
|
Field & number = ast->as<ASTLiteral &>().value;
|
||||||
|
|
||||||
|
if ((number.getType() == Field::Types::UInt64 && type_info.isUInt64()) ||
|
||||||
|
(number.getType() == Field::Types::Int64 && type_info.isInt64()) ||
|
||||||
|
(number.getType() == Field::Types::Float64 && type_info.isFloat64()) ||
|
||||||
|
is_nullable)
|
||||||
|
{
|
||||||
|
columns[column_idx]->insert(number);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw DB::Exception("Cannot parse literal: type mismatch", ErrorCodes::CANNOT_PARSE_EXPRESSION_USING_TEMPLATE);
|
||||||
|
}
|
||||||
|
|
||||||
ColumnPtr ConstantExpressionTemplate::evaluateAll()
|
ColumnPtr ConstantExpressionTemplate::evaluateAll()
|
||||||
{
|
{
|
||||||
Block evaluated = literals.cloneWithColumns(std::move(columns));
|
Block evaluated = literals.cloneWithColumns(std::move(columns));
|
||||||
@ -253,13 +360,14 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll()
|
|||||||
evaluated.insert({ColumnConst::create(ColumnUInt8::create(1, 0), rows_count), std::make_shared<DataTypeUInt8>(), "_dummy"});
|
evaluated.insert({ColumnConst::create(ColumnUInt8::create(1, 0), rows_count), std::make_shared<DataTypeUInt8>(), "_dummy"});
|
||||||
actions_on_literals->execute(evaluated);
|
actions_on_literals->execute(evaluated);
|
||||||
|
|
||||||
if (!evaluated || evaluated.rows() == 0)
|
if (!evaluated || evaluated.rows() != rows_count)
|
||||||
throw Exception("Logical error: empty block after evaluation of batch of constant expressions",
|
throw Exception("Number of rows mismatch after evaluation of batch of constant expressions: got " +
|
||||||
|
std::to_string(evaluated.rows()) + " rows for " + std::to_string(rows_count) + " expressions",
|
||||||
ErrorCodes::LOGICAL_ERROR);
|
ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
if (!evaluated.has(result_column_name))
|
if (!evaluated.has(result_column_name))
|
||||||
throw Exception("Cannot evaluate template " + result_column_name + ", block structure:\n" + evaluated.dumpStructure(),
|
throw Exception("Cannot evaluate template " + result_column_name + ", block structure:\n" + evaluated.dumpStructure(),
|
||||||
ErrorCodes::CANNOT_EVALUATE_EXPRESSION_TEMPLATE);
|
ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
rows_count = 0;
|
rows_count = 0;
|
||||||
return evaluated.getByName(result_column_name).column->convertToFullColumnIfConst();
|
return evaluated.getByName(result_column_name).column->convertToFullColumnIfConst();
|
||||||
|
@ -8,22 +8,36 @@
|
|||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
struct LiteralInfo;
|
||||||
|
|
||||||
|
/// Deduces template of an expression by replacing literals with dummy columns.
|
||||||
|
/// It allows to parse and evaluate similar expressions without using heavy IParsers and ExpressionAnalyzer.
|
||||||
|
/// Using ConstantExpressionTemplate for one expression is slower then evaluateConstantExpression(...),
|
||||||
|
/// but it's significantly faster for batch of expressions
|
||||||
class ConstantExpressionTemplate
|
class ConstantExpressionTemplate
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ConstantExpressionTemplate(const IDataType & result_column_type, TokenIterator expression_begin, TokenIterator expression_end,
|
/// Deduce template of expression of type result_column_type
|
||||||
|
ConstantExpressionTemplate(DataTypePtr result_column_type_, TokenIterator expression_begin, TokenIterator expression_end,
|
||||||
const ASTPtr & expression, const Context & context);
|
const ASTPtr & expression, const Context & context);
|
||||||
|
|
||||||
|
/// Read expression from istr, assert it has the same structure and the same types of literals (template matches)
|
||||||
|
/// and parse literals into temporary columns
|
||||||
void parseExpression(ReadBuffer & istr, const FormatSettings & settings);
|
void parseExpression(ReadBuffer & istr, const FormatSettings & settings);
|
||||||
|
|
||||||
|
/// Evaluate batch of expressions were parsed using template
|
||||||
ColumnPtr evaluateAll();
|
ColumnPtr evaluateAll();
|
||||||
|
|
||||||
size_t rowsCount() const { return rows_count; }
|
size_t rowsCount() const { return rows_count; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr);
|
static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr);
|
||||||
|
bool getDataType(const LiteralInfo & info, DataTypePtr & type) const;
|
||||||
|
void parseLiteralAndAssertType(ReadBuffer & istr, const IDataType & type, size_t column_idx);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
DataTypePtr result_column_type;
|
||||||
|
|
||||||
std::vector<String> tokens;
|
std::vector<String> tokens;
|
||||||
std::vector<size_t> token_after_literal_idx;
|
std::vector<size_t> token_after_literal_idx;
|
||||||
|
|
||||||
@ -32,7 +46,7 @@ private:
|
|||||||
Block literals;
|
Block literals;
|
||||||
MutableColumns columns;
|
MutableColumns columns;
|
||||||
|
|
||||||
std::vector<char> need_special_parser;
|
std::vector<char> use_special_parser;
|
||||||
|
|
||||||
/// For expressions without literals (e.g. "now()")
|
/// For expressions without literals (e.g. "now()")
|
||||||
size_t rows_count = 0;
|
size_t rows_count = 0;
|
||||||
|
@ -25,9 +25,7 @@ namespace ErrorCodes
|
|||||||
extern const int CANNOT_PARSE_DATE;
|
extern const int CANNOT_PARSE_DATE;
|
||||||
extern const int SYNTAX_ERROR;
|
extern const int SYNTAX_ERROR;
|
||||||
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
|
extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE;
|
||||||
extern const int CANNOT_CREATE_EXPRESSION_TEMPLATE;
|
|
||||||
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE;
|
extern const int CANNOT_PARSE_EXPRESSION_USING_TEMPLATE;
|
||||||
extern const int CANNOT_EVALUATE_EXPRESSION_TEMPLATE;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -233,7 +231,7 @@ ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx, boo
|
|||||||
ErrorCodes::LOGICAL_ERROR);
|
ErrorCodes::LOGICAL_ERROR);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
templates[column_idx] = ConstantExpressionTemplate(type, TokenIterator(tokens), token_iterator, ast, *context);
|
templates[column_idx] = ConstantExpressionTemplate(header.getByPosition(column_idx).type, TokenIterator(tokens), token_iterator, ast, *context);
|
||||||
buf.rollbackToCheckpoint();
|
buf.rollbackToCheckpoint();
|
||||||
templates[column_idx].value().parseExpression(buf, format_settings);
|
templates[column_idx].value().parseExpression(buf, format_settings);
|
||||||
assertDelimiterAfterValue(column_idx);
|
assertDelimiterAfterValue(column_idx);
|
||||||
|
Loading…
Reference in New Issue
Block a user