thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Miscellaneous [#METR-2944].

Browse Source
This commit is contained in:
Alexey Milovidov 2017-01-06 19:34:15 +03:00
parent 2f7ff07e1f
commit ae7e7e90eb
4 changed files with 183 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
dbms/include/DB/Core/FieldVisitors.h
View File

@ -2,12 +2,8 @@
#include <DB/Core/Field.h> #include <DB/Core/Field.h>
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/WriteBuffer.h> class SipHash;
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/WriteBufferFromString.h>
namespace DB namespace DB
@ -19,9 +15,9 @@ namespace ErrorCodes
} }
/** StaticVisitor (его наследники) - класс с перегруженными для разных типов операторами (). /** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
* Вызвать visitor для field можно с помощью функции apply_visitor. * You could call visitor for field using function 'apply_visitor'.
* Также поддерживается visitor, в котором оператор () принимает два аргумента. * Also "binary visitor" is supported - its operator() takes two arguments.
*/ */
template <typename R = void> template <typename R = void>
struct StaticVisitor struct StaticVisitor
@ -30,65 +26,38 @@ struct StaticVisitor
}; };
/// F is template parameter, to allow universal reference for field, that is useful for const and non-const values.
template <typename Visitor, typename F> template <typename Visitor, typename F>
typename Visitor::ResultType apply_visitor_impl(Visitor & visitor, F & field) typename Visitor::ResultType apply_visitor(Visitor && visitor, F && field)
{ {
switch (field.getType()) switch (field.getType())
{ {
case Field::Types::Null: return visitor(field.template get<Null>()); case Field::Types::Null: return visitor(field.template get<Null>());
case Field::Types::UInt64: return visitor(field.template get<UInt64>()); case Field::Types::UInt64: return visitor(field.template get<UInt64>());
case Field::Types::Int64: return visitor(field.template get<Int64>()); case Field::Types::Int64: return visitor(field.template get<Int64>());
case Field::Types::Float64: return visitor(field.template get<Float64>()); case Field::Types::Float64: return visitor(field.template get<Float64>());
case Field::Types::String: return visitor(field.template get<String>()); case Field::Types::String: return visitor(field.template get<String>());
case Field::Types::Array: return visitor(field.template get<Array>()); case Field::Types::Array: return visitor(field.template get<Array>());
case Field::Types::Tuple: return visitor(field.template get<Tuple>()); case Field::Types::Tuple: return visitor(field.template get<Tuple>());
default: default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
} }
} }
/** Эти штуки нужны, чтобы принимать временный объект по константной ссылке.
* В шаблон выше, типы форвардятся уже с const-ом.
*/
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor, typename F1, typename F2> template <typename Visitor, typename F1, typename F2>
typename Visitor::ResultType apply_binary_visitor_impl2(Visitor & visitor, F1 & field1, F2 & field2) static typename Visitor::ResultType apply_binary_visitor_impl(Visitor && visitor, F1 && field1, F2 && field2)
{ {
switch (field2.getType()) switch (field2.getType())
{ {
case Field::Types::Null: return visitor(field1, field2.template get<Null>()); case Field::Types::Null: return visitor(field1, field2.template get<Null>());
case Field::Types::UInt64: return visitor(field1, field2.template get<UInt64>()); case Field::Types::UInt64: return visitor(field1, field2.template get<UInt64>());
case Field::Types::Int64: return visitor(field1, field2.template get<Int64>()); case Field::Types::Int64: return visitor(field1, field2.template get<Int64>());
case Field::Types::Float64: return visitor(field1, field2.template get<Float64>()); case Field::Types::Float64: return visitor(field1, field2.template get<Float64>());
case Field::Types::String: return visitor(field1, field2.template get<String>()); case Field::Types::String: return visitor(field1, field2.template get<String>());
case Field::Types::Array: return visitor(field1, field2.template get<Array>()); case Field::Types::Array: return visitor(field1, field2.template get<Array>());
case Field::Types::Tuple: return visitor(field1, field2.template get<Tuple>()); case Field::Types::Tuple: return visitor(field1, field2.template get<Tuple>());
default: default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -96,130 +65,67 @@ typename Visitor::ResultType apply_binary_visitor_impl2(Visitor & visitor, F1 &
} }
template <typename Visitor, typename F1, typename F2> template <typename Visitor, typename F1, typename F2>
typename Visitor::ResultType apply_binary_visitor_impl1(Visitor & visitor, F1 & field1, F2 & field2) typename Visitor::ResultType apply_visitor(Visitor && visitor, F1 && field1, F2 && field2)
{ {
switch (field1.getType()) switch (field1.getType())
{ {
case Field::Types::Null: return apply_binary_visitor_impl2(visitor, field1.template get<Null>(), field2); case Field::Types::Null:
case Field::Types::UInt64: return apply_binary_visitor_impl2(visitor, field1.template get<UInt64>(), field2); return apply_binary_visitor_impl(
case Field::Types::Int64: return apply_binary_visitor_impl2(visitor, field1.template get<Int64>(), field2); std::forward<Visitor>(visitor), field1.template get<Null>(), std::forward<Field>(field2));
case Field::Types::Float64: return apply_binary_visitor_impl2(visitor, field1.template get<Float64>(), field2); case Field::Types::UInt64:
case Field::Types::String: return apply_binary_visitor_impl2(visitor, field1.template get<String>(), field2); return apply_binary_visitor_impl(
case Field::Types::Array: return apply_binary_visitor_impl2(visitor, field1.template get<Array>(), field2); std::forward<Visitor>(visitor), field1.template get<UInt64>(), std::forward<Field>(field2));
case Field::Types::Tuple: return apply_binary_visitor_impl2(visitor, field1.template get<Tuple>(), field2); case Field::Types::Int64:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Int64>(), std::forward<Field>(field2));
case Field::Types::Float64:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Float64>(), std::forward<Field>(field2));
case Field::Types::String:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<String>(), std::forward<Field>(field2));
case Field::Types::Array:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Array>(), std::forward<Field>(field2));
case Field::Types::Tuple:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Tuple>(), std::forward<Field>(field2));
default: default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
} }
} }
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor> /** Prints Field as literal in SQL query */
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field1, const Field & field2) class FieldVisitorToString : public StaticVisitor<String>
{ {
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
/** Возвращает строковый дамп типа */
class FieldVisitorDump : public StaticVisitor<String>
{
private:
template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
String res;
WriteBufferFromString wb(res);
wb.write(prefix, strlen(prefix));
writeQuoted(x, wb);
return res;
}
public: public:
String operator() (const Null & x) const { return "NULL"; } String operator() (const Null & x) const;
String operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); } String operator() (const UInt64 & x) const;
String operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); } String operator() (const Int64 & x) const;
String operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); } String operator() (const Float64 & x) const;
String operator() (const String & x) const; String operator() (const String & x) const;
String operator() (const Array & x) const; String operator() (const Array & x) const;
String operator() (const Tuple & x) const; String operator() (const Tuple & x) const;
}; };
/** Выводит текстовое представление типа, как литерала в SQL запросе */ /** Print readable and unique text dump of field type and value. */
class FieldVisitorToString : public StaticVisitor<String> class FieldVisitorDump : public StaticVisitor<String>
{ {
private:
template <typename T>
static inline String formatQuoted(T x)
{
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
/** В отличие от writeFloatText (и writeQuoted), если число после форматирования выглядит целым, всё равно добавляет десятичную точку.
* - для того, чтобы это число могло обратно распарситься как Float64 парсером запроса (иначе распарсится как целое).
*
* При этом, не оставляет завершающие нули справа.
*
* NOTE: При таком roundtrip-е, точность может теряться.
*/
static String formatFloat(const Float64 x);
public: public:
String operator() (const Null & x) const { return "NULL"; } String operator() (const Null & x) const;
String operator() (const UInt64 & x) const { return formatQuoted(x); } String operator() (const UInt64 & x) const;
String operator() (const Int64 & x) const { return formatQuoted(x); } String operator() (const Int64 & x) const;
String operator() (const Float64 & x) const { return formatFloat(x); } String operator() (const Float64 & x) const;
String operator() (const String & x) const { return formatQuoted(x); } String operator() (const String & x) const;
String operator() (const Array & x) const; String operator() (const Array & x) const;
String operator() (const Tuple & x) const; String operator() (const Tuple & x) const;
}; };
/** Числовой тип преобразует в указанный. */ /** Converts numberic value of any type to specified type. */
template <typename T> template <typename T>
class FieldVisitorConvertToNumber : public StaticVisitor<T> class FieldVisitorConvertToNumber : public StaticVisitor<T>
{ {
@ -250,6 +156,23 @@ public:
}; };
/** Updates SipHash by type and value of Field */
class FieldVisitorHash : public StaticVisitor<>
{
private:
SipHash & hash;
public:
FieldVisitorHash(SipHash & hash);
void operator() (const Null & x) const;
void operator() (const UInt64 & x) const;
void operator() (const Int64 & x) const;
void operator() (const Float64 & x) const;
void operator() (const String & x) const;
void operator() (const Array & x) const;
};
/// Converts string with date or datetime (in format 'YYYY-MM-DD hh:mm:ss') to UInt64 containing numeric value of date (or datetime) /// Converts string with date or datetime (in format 'YYYY-MM-DD hh:mm:ss') to UInt64 containing numeric value of date (or datetime)
UInt64 stringToDateOrDateTime(const String & s); UInt64 stringToDateOrDateTime(const String & s);

5
dbms/include/DB/Functions/FunctionsComparison.h
View File

@ -12,10 +12,13 @@
#include <DB/DataTypes/DataTypeString.h> #include <DB/DataTypes/DataTypeString.h>
#include <DB/DataTypes/DataTypeFixedString.h> #include <DB/DataTypes/DataTypeFixedString.h>
#include <DB/DataTypes/DataTypeTuple.h> #include <DB/DataTypes/DataTypeTuple.h>
#include <DB/DataTypes/DataTypeEnum.h>
#include <DB/Functions/FunctionsLogical.h> #include <DB/Functions/FunctionsLogical.h>
#include <DB/Functions/IFunction.h> #include <DB/Functions/IFunction.h>
#include <DB/DataTypes/DataTypeEnum.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/ReadHelpers.h>
namespace DB namespace DB

110
dbms/src/Core/FieldVisitors.cpp
View File

@ -1,3 +1,9 @@
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/WriteBufferFromString.h>
#include <DB/Core/FieldVisitors.h> #include <DB/Core/FieldVisitors.h>
@ -5,7 +11,8 @@ namespace DB
{ {
String FieldVisitorDump::operator() (const String & x) const template <typename T>
static inline String formatQuoted(T x)
{ {
String res; String res;
WriteBufferFromString wb(res); WriteBufferFromString wb(res);
@ -13,7 +20,32 @@ String FieldVisitorDump::operator() (const String & x) const
return res; return res;
} }
String FieldVisitorDump::operator() (const Array & x) const template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
String res;
WriteBufferFromString wb(res);
wb.write(prefix, strlen(prefix));
writeQuoted(x, wb);
return res;
}
String FieldVisitorDump::operator() (const Null & x) const { return "NULL"; }
String FieldVisitorDump::operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); }
String FieldVisitorDump::operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); }
String FieldVisitorDump::operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); }
String FieldVisitorDump::operator() (const String & x) const
{
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
String FieldVisitorDump::operator() (const Array & x) const
{ {
String res; String res;
WriteBufferFromString wb(res); WriteBufferFromString wb(res);
@ -31,7 +63,7 @@ String FieldVisitorDump::operator() (const Array & x) const
return res; return res;
} }
String FieldVisitorDump::operator() (const Tuple & x_def) const String FieldVisitorDump::operator() (const Tuple & x_def) const
{ {
auto & x = x_def.t; auto & x = x_def.t;
String res; String res;
@ -51,7 +83,15 @@ String FieldVisitorDump::operator() (const Tuple & x_def) const
} }
String FieldVisitorToString::formatFloat(const Float64 x) /** In contrast to writeFloatText (and writeQuoted),
* even if number looks like integer after formatting, prints decimal point nevertheless (for example, Float64(1) is printed as 1.).
* - because resulting text must be able to be parsed back as Float64 by query parser (otherwise it will be parsed as integer).
*
* Trailing zeros after decimal point are omitted.
*
* NOTE: Roundtrip may lead to loss of precision.
*/
static String formatFloat(const Float64 x)
{ {
DoubleConverter<true>::BufferType buffer; DoubleConverter<true>::BufferType buffer;
double_conversion::StringBuilder builder{buffer, sizeof(buffer)}; double_conversion::StringBuilder builder{buffer, sizeof(buffer)};
@ -64,7 +104,15 @@ String FieldVisitorToString::formatFloat(const Float64 x)
return { buffer, buffer + builder.position() }; return { buffer, buffer + builder.position() };
} }
String FieldVisitorToString::operator() (const Array & x) const
String FieldVisitorToString::operator() (const Null & x) const { return "NULL"; }
String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Float64 & x) const { return formatFloat(x); }
String FieldVisitorToString::operator() (const String & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Array & x) const
{ {
String res; String res;
WriteBufferFromString wb(res); WriteBufferFromString wb(res);
@ -82,7 +130,7 @@ String FieldVisitorToString::operator() (const Array & x) const
return res; return res;
} }
String FieldVisitorToString::operator() (const Tuple & x_def) const String FieldVisitorToString::operator() (const Tuple & x_def) const
{ {
auto & x = x_def.t; auto & x = x_def.t;
String res; String res;
@ -102,6 +150,56 @@ String FieldVisitorToString::operator() (const Tuple & x_def) const
} }
FieldVisitorHash::FieldVisitorHash(SipHash & hash) : hash(hash) {}
void FieldVisitorHash::operator() (const Null & x) const
{
UInt8 type = Field::Types::Null;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
}
void FieldVisitorHash::operator() (const UInt64 & x) const
{
UInt8 type = Field::Types::UInt64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const Int64 & x) const
{
UInt8 type = Field::Types::Int64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const String & x) const
{
UInt8 type = Field::Types::String;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
hash.update(x.data(), x.size());
}
void FieldVisitorHash::operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
for (const auto & elem : x)
apply_visitor(*this, elem);
}
UInt64 stringToDateOrDateTime(const String & s) UInt64 stringToDateOrDateTime(const String & s)
{ {
if (s.size() == strlen("YYYY-MM-DD")) if (s.size() == strlen("YYYY-MM-DD"))

57
dbms/src/Parsers/ASTLiteral.cpp
View File

@ -7,63 +7,6 @@ namespace DB
{ {
/** Обновляет SipHash по данным Field */
class FieldVisitorHash : public StaticVisitor<>
{
private:
SipHash & hash;
public:
FieldVisitorHash(SipHash & hash) : hash(hash) {}
void operator() (const Null & x) const
{
UInt8 type = Field::Types::Null;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
}
void operator() (const UInt64 & x) const
{
UInt8 type = Field::Types::UInt64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const Int64 & x) const
{
UInt8 type = Field::Types::Int64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const String & x) const
{
UInt8 type = Field::Types::String;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
hash.update(x.data(), x.size());
}
void operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
for (const auto & elem : x)
apply_visitor(*this, elem);
}
};
String ASTLiteral::getColumnName() const String ASTLiteral::getColumnName() const
{ {
/// Отдельный случай для очень больших массивов. Вместо указания всех элементов, будем использовать хэш от содержимого. /// Отдельный случай для очень больших массивов. Вместо указания всех элементов, будем использовать хэш от содержимого.