Miscellaneous [#METR-2944].

This commit is contained in:
Alexey Milovidov 2017-01-06 19:34:15 +03:00
parent 2f7ff07e1f
commit ae7e7e90eb
4 changed files with 183 additions and 216 deletions

View File

@ -2,12 +2,8 @@
#include <DB/Core/Field.h>
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/WriteBufferFromString.h>
class SipHash;
namespace DB
@ -19,9 +15,9 @@ namespace ErrorCodes
}
/** StaticVisitor (его наследники) - класс с перегруженными для разных типов операторами ().
* Вызвать visitor для field можно с помощью функции apply_visitor.
* Также поддерживается visitor, в котором оператор () принимает два аргумента.
/** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
* You could call visitor for field using function 'apply_visitor'.
* Also "binary visitor" is supported - its operator() takes two arguments.
*/
template <typename R = void>
struct StaticVisitor
@ -30,65 +26,38 @@ struct StaticVisitor
};
/// F is template parameter, to allow universal reference for field, that is useful for const and non-const values.
template <typename Visitor, typename F>
typename Visitor::ResultType apply_visitor_impl(Visitor & visitor, F & field)
typename Visitor::ResultType apply_visitor(Visitor && visitor, F && field)
{
switch (field.getType())
{
case Field::Types::Null: return visitor(field.template get<Null>());
case Field::Types::UInt64: return visitor(field.template get<UInt64>());
case Field::Types::Int64: return visitor(field.template get<Int64>());
case Field::Types::Float64: return visitor(field.template get<Float64>());
case Field::Types::String: return visitor(field.template get<String>());
case Field::Types::Array: return visitor(field.template get<Array>());
case Field::Types::Tuple: return visitor(field.template get<Tuple>());
case Field::Types::Null: return visitor(field.template get<Null>());
case Field::Types::UInt64: return visitor(field.template get<UInt64>());
case Field::Types::Int64: return visitor(field.template get<Int64>());
case Field::Types::Float64: return visitor(field.template get<Float64>());
case Field::Types::String: return visitor(field.template get<String>());
case Field::Types::Array: return visitor(field.template get<Array>());
case Field::Types::Tuple: return visitor(field.template get<Tuple>());
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
}
/** Эти штуки нужны, чтобы принимать временный объект по константной ссылке.
* В шаблон выше, типы форвардятся уже с const-ом.
*/
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field)
{
return apply_visitor_impl(visitor, field);
}
template <typename Visitor, typename F1, typename F2>
typename Visitor::ResultType apply_binary_visitor_impl2(Visitor & visitor, F1 & field1, F2 & field2)
static typename Visitor::ResultType apply_binary_visitor_impl(Visitor && visitor, F1 && field1, F2 && field2)
{
switch (field2.getType())
{
case Field::Types::Null: return visitor(field1, field2.template get<Null>());
case Field::Types::UInt64: return visitor(field1, field2.template get<UInt64>());
case Field::Types::Int64: return visitor(field1, field2.template get<Int64>());
case Field::Types::Float64: return visitor(field1, field2.template get<Float64>());
case Field::Types::String: return visitor(field1, field2.template get<String>());
case Field::Types::Array: return visitor(field1, field2.template get<Array>());
case Field::Types::Tuple: return visitor(field1, field2.template get<Tuple>());
case Field::Types::Null: return visitor(field1, field2.template get<Null>());
case Field::Types::UInt64: return visitor(field1, field2.template get<UInt64>());
case Field::Types::Int64: return visitor(field1, field2.template get<Int64>());
case Field::Types::Float64: return visitor(field1, field2.template get<Float64>());
case Field::Types::String: return visitor(field1, field2.template get<String>());
case Field::Types::Array: return visitor(field1, field2.template get<Array>());
case Field::Types::Tuple: return visitor(field1, field2.template get<Tuple>());
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -96,130 +65,67 @@ typename Visitor::ResultType apply_binary_visitor_impl2(Visitor & visitor, F1 &
}
template <typename Visitor, typename F1, typename F2>
typename Visitor::ResultType apply_binary_visitor_impl1(Visitor & visitor, F1 & field1, F2 & field2)
typename Visitor::ResultType apply_visitor(Visitor && visitor, F1 && field1, F2 && field2)
{
switch (field1.getType())
{
case Field::Types::Null: return apply_binary_visitor_impl2(visitor, field1.template get<Null>(), field2);
case Field::Types::UInt64: return apply_binary_visitor_impl2(visitor, field1.template get<UInt64>(), field2);
case Field::Types::Int64: return apply_binary_visitor_impl2(visitor, field1.template get<Int64>(), field2);
case Field::Types::Float64: return apply_binary_visitor_impl2(visitor, field1.template get<Float64>(), field2);
case Field::Types::String: return apply_binary_visitor_impl2(visitor, field1.template get<String>(), field2);
case Field::Types::Array: return apply_binary_visitor_impl2(visitor, field1.template get<Array>(), field2);
case Field::Types::Tuple: return apply_binary_visitor_impl2(visitor, field1.template get<Tuple>(), field2);
case Field::Types::Null:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Null>(), std::forward<Field>(field2));
case Field::Types::UInt64:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<UInt64>(), std::forward<Field>(field2));
case Field::Types::Int64:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Int64>(), std::forward<Field>(field2));
case Field::Types::Float64:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Float64>(), std::forward<Field>(field2));
case Field::Types::String:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<String>(), std::forward<Field>(field2));
case Field::Types::Array:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Array>(), std::forward<Field>(field2));
case Field::Types::Tuple:
return apply_binary_visitor_impl(
std::forward<Visitor>(visitor), field1.template get<Tuple>(), std::forward<Field>(field2));
default:
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
}
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, Field & field1, const Field & field2)
/** Prints Field as literal in SQL query */
class FieldVisitorToString : public StaticVisitor<String>
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(Visitor & visitor, const Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field1, Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
template <typename Visitor>
typename Visitor::ResultType apply_visitor(const Visitor & visitor, const Field & field1, const Field & field2)
{
return apply_binary_visitor_impl1(visitor, field1, field2);
}
/** Возвращает строковый дамп типа */
class FieldVisitorDump : public StaticVisitor<String>
{
private:
template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
String res;
WriteBufferFromString wb(res);
wb.write(prefix, strlen(prefix));
writeQuoted(x, wb);
return res;
}
public:
String operator() (const Null & x) const { return "NULL"; }
String operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); }
String operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); }
String operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); }
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
String operator() (const Tuple & x) const;
};
/** Выводит текстовое представление типа, как литерала в SQL запросе */
class FieldVisitorToString : public StaticVisitor<String>
/** Print readable and unique text dump of field type and value. */
class FieldVisitorDump : public StaticVisitor<String>
{
private:
template <typename T>
static inline String formatQuoted(T x)
{
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
/** В отличие от writeFloatText (и writeQuoted), если число после форматирования выглядит целым, всё равно добавляет десятичную точку.
* - для того, чтобы это число могло обратно распарситься как Float64 парсером запроса (иначе распарсится как целое).
*
* При этом, не оставляет завершающие нули справа.
*
* NOTE: При таком roundtrip-е, точность может теряться.
*/
static String formatFloat(const Float64 x);
public:
String operator() (const Null & x) const { return "NULL"; }
String operator() (const UInt64 & x) const { return formatQuoted(x); }
String operator() (const Int64 & x) const { return formatQuoted(x); }
String operator() (const Float64 & x) const { return formatFloat(x); }
String operator() (const String & x) const { return formatQuoted(x); }
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
String operator() (const Tuple & x) const;
};
/** Числовой тип преобразует в указанный. */
/** Converts numberic value of any type to specified type. */
template <typename T>
class FieldVisitorConvertToNumber : public StaticVisitor<T>
{
@ -250,6 +156,23 @@ public:
};
/** Updates SipHash by type and value of Field */
class FieldVisitorHash : public StaticVisitor<>
{
private:
SipHash & hash;
public:
FieldVisitorHash(SipHash & hash);
void operator() (const Null & x) const;
void operator() (const UInt64 & x) const;
void operator() (const Int64 & x) const;
void operator() (const Float64 & x) const;
void operator() (const String & x) const;
void operator() (const Array & x) const;
};
/// Converts string with date or datetime (in format 'YYYY-MM-DD hh:mm:ss') to UInt64 containing numeric value of date (or datetime)
UInt64 stringToDateOrDateTime(const String & s);

View File

@ -12,10 +12,13 @@
#include <DB/DataTypes/DataTypeString.h>
#include <DB/DataTypes/DataTypeFixedString.h>
#include <DB/DataTypes/DataTypeTuple.h>
#include <DB/DataTypes/DataTypeEnum.h>
#include <DB/Functions/FunctionsLogical.h>
#include <DB/Functions/IFunction.h>
#include <DB/DataTypes/DataTypeEnum.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/ReadHelpers.h>
namespace DB

View File

@ -1,3 +1,9 @@
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ReadBufferFromString.h>
#include <DB/IO/WriteBufferFromString.h>
#include <DB/Core/FieldVisitors.h>
@ -5,7 +11,8 @@ namespace DB
{
String FieldVisitorDump::operator() (const String & x) const
template <typename T>
static inline String formatQuoted(T x)
{
String res;
WriteBufferFromString wb(res);
@ -13,7 +20,32 @@ String FieldVisitorDump::operator() (const String & x) const
return res;
}
String FieldVisitorDump::operator() (const Array & x) const
template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
String res;
WriteBufferFromString wb(res);
wb.write(prefix, strlen(prefix));
writeQuoted(x, wb);
return res;
}
String FieldVisitorDump::operator() (const Null & x) const { return "NULL"; }
String FieldVisitorDump::operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); }
String FieldVisitorDump::operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); }
String FieldVisitorDump::operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); }
String FieldVisitorDump::operator() (const String & x) const
{
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
String FieldVisitorDump::operator() (const Array & x) const
{
String res;
WriteBufferFromString wb(res);
@ -31,7 +63,7 @@ String FieldVisitorDump::operator() (const Array & x) const
return res;
}
String FieldVisitorDump::operator() (const Tuple & x_def) const
String FieldVisitorDump::operator() (const Tuple & x_def) const
{
auto & x = x_def.t;
String res;
@ -51,7 +83,15 @@ String FieldVisitorDump::operator() (const Tuple & x_def) const
}
String FieldVisitorToString::formatFloat(const Float64 x)
/** In contrast to writeFloatText (and writeQuoted),
* even if number looks like integer after formatting, prints decimal point nevertheless (for example, Float64(1) is printed as 1.).
* - because resulting text must be able to be parsed back as Float64 by query parser (otherwise it will be parsed as integer).
*
* Trailing zeros after decimal point are omitted.
*
* NOTE: Roundtrip may lead to loss of precision.
*/
static String formatFloat(const Float64 x)
{
DoubleConverter<true>::BufferType buffer;
double_conversion::StringBuilder builder{buffer, sizeof(buffer)};
@ -64,7 +104,15 @@ String FieldVisitorToString::formatFloat(const Float64 x)
return { buffer, buffer + builder.position() };
}
String FieldVisitorToString::operator() (const Array & x) const
String FieldVisitorToString::operator() (const Null & x) const { return "NULL"; }
String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Float64 & x) const { return formatFloat(x); }
String FieldVisitorToString::operator() (const String & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Array & x) const
{
String res;
WriteBufferFromString wb(res);
@ -82,7 +130,7 @@ String FieldVisitorToString::operator() (const Array & x) const
return res;
}
String FieldVisitorToString::operator() (const Tuple & x_def) const
String FieldVisitorToString::operator() (const Tuple & x_def) const
{
auto & x = x_def.t;
String res;
@ -102,6 +150,56 @@ String FieldVisitorToString::operator() (const Tuple & x_def) const
}
FieldVisitorHash::FieldVisitorHash(SipHash & hash) : hash(hash) {}
void FieldVisitorHash::operator() (const Null & x) const
{
UInt8 type = Field::Types::Null;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
}
void FieldVisitorHash::operator() (const UInt64 & x) const
{
UInt8 type = Field::Types::UInt64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const Int64 & x) const
{
UInt8 type = Field::Types::Int64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void FieldVisitorHash::operator() (const String & x) const
{
UInt8 type = Field::Types::String;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
hash.update(x.data(), x.size());
}
void FieldVisitorHash::operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
for (const auto & elem : x)
apply_visitor(*this, elem);
}
UInt64 stringToDateOrDateTime(const String & s)
{
if (s.size() == strlen("YYYY-MM-DD"))

View File

@ -7,63 +7,6 @@ namespace DB
{
/** Обновляет SipHash по данным Field */
class FieldVisitorHash : public StaticVisitor<>
{
private:
SipHash & hash;
public:
FieldVisitorHash(SipHash & hash) : hash(hash) {}
void operator() (const Null & x) const
{
UInt8 type = Field::Types::Null;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
}
void operator() (const UInt64 & x) const
{
UInt8 type = Field::Types::UInt64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const Int64 & x) const
{
UInt8 type = Field::Types::Int64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
hash.update(reinterpret_cast<const char *>(&x), sizeof(x));
}
void operator() (const String & x) const
{
UInt8 type = Field::Types::String;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
hash.update(x.data(), x.size());
}
void operator() (const Array & x) const
{
UInt8 type = Field::Types::Array;
hash.update(reinterpret_cast<const char *>(&type), sizeof(type));
size_t size = x.size();
hash.update(reinterpret_cast<const char *>(&size), sizeof(size));
for (const auto & elem : x)
apply_visitor(*this, elem);
}
};
String ASTLiteral::getColumnName() const
{
/// Отдельный случай для очень больших массивов. Вместо указания всех элементов, будем использовать хэш от содержимого.