ClickHouse/dbms/include/DB/IO/WriteHelpers.h

605 lines
18 KiB
C
Raw Normal View History

2011-10-10 06:10:49 +00:00
#pragma once
2010-06-04 18:25:25 +00:00
#include <cstring>
#include <cstdio>
#include <limits>
#include <algorithm>
#include <Yandex/Common.h>
2011-05-05 19:10:17 +00:00
#include <Yandex/DateLUT.h>
#include <mysqlxx/Row.h>
2010-06-04 18:25:25 +00:00
#include <DB/Core/Types.h>
#include <DB/Core/Exception.h>
#include <DB/Core/ErrorCodes.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/WriteIntText.h>
2011-08-19 18:31:14 +00:00
#include <DB/IO/VarInt.h>
2013-06-21 20:34:19 +00:00
#include <DB/IO/WriteBufferFromString.h>
2014-04-02 13:45:39 +00:00
#include <city.h>
2010-06-04 18:25:25 +00:00
2010-06-07 17:14:13 +00:00
#define WRITE_HELPERS_DEFAULT_FLOAT_PRECISION 6U
2010-06-04 18:25:25 +00:00
namespace DB
{
/// Функции-помошники для форматированной записи
inline void writeChar(char x, WriteBuffer & buf)
{
buf.nextIfAtEnd();
*buf.position() = x;
++buf.position();
}
/// Запись POD-типа в native формате
2011-07-04 18:22:37 +00:00
template <typename T>
inline void writePODBinary(const T & x, WriteBuffer & buf)
2011-07-04 18:22:37 +00:00
{
buf.write(reinterpret_cast<const char *>(&x), sizeof(x));
}
2011-08-09 17:24:17 +00:00
template <typename T>
inline void writeIntBinary(const T & x, WriteBuffer & buf)
2011-08-09 17:24:17 +00:00
{
writePODBinary(x, buf);
2011-08-09 17:24:17 +00:00
}
template <typename T>
inline void writeFloatBinary(const T & x, WriteBuffer & buf)
2011-08-09 17:24:17 +00:00
{
writePODBinary(x, buf);
2011-08-09 17:24:17 +00:00
}
2011-07-04 18:22:37 +00:00
2013-10-10 23:06:51 +00:00
inline void writeStringBinary(const std::string & s, WriteBuffer & buf)
2011-08-19 18:31:14 +00:00
{
writeVarUInt(s.size(), buf);
buf.write(s.data(), s.size());
}
inline void writeStringBinary(const char * s, WriteBuffer & buf)
{
writeVarUInt(strlen(s), buf);
buf.write(s, strlen(s));
}
2011-08-19 18:31:14 +00:00
template <typename T>
2013-10-10 23:06:51 +00:00
void writeVectorBinary(const std::vector<T> & v, WriteBuffer & buf)
{
writeVarUInt(v.size(), buf);
for (typename std::vector<T>::const_iterator it = v.begin(); it != v.end(); ++it)
writeBinary(*it, buf);
}
inline void writeBoolText(bool x, WriteBuffer & buf)
{
writeChar(x ? '1' : '0', buf);
}
2010-06-04 18:25:25 +00:00
template <typename T>
void writeFloatText(T x, WriteBuffer & buf, unsigned precision = WRITE_HELPERS_DEFAULT_FLOAT_PRECISION)
{
2012-04-13 19:50:42 +00:00
char tmp[24];
int res = std::snprintf(tmp, 24, "%.*g", precision, x);
2010-06-04 18:25:25 +00:00
2012-04-13 19:50:42 +00:00
if (res >= 24 || res <= 0)
2010-06-04 18:25:25 +00:00
throw Exception("Cannot print float or double number", ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER);
2010-06-04 18:34:25 +00:00
buf.write(tmp, res);
2010-06-04 18:25:25 +00:00
}
2011-08-09 17:24:17 +00:00
2010-06-04 18:25:25 +00:00
inline void writeString(const String & s, WriteBuffer & buf)
{
buf.write(s.data(), s.size());
}
inline void writeString(const char * data, size_t size, WriteBuffer & buf)
{
buf.write(data, size);
}
2013-05-20 09:33:45 +00:00
/** Пишет С-строку без создания временного объекта. Если строка - литерал, то strlen выполняется на этапе компиляции.
* Используйте, когда строка - литерал.
*/
#define writeCString(s, buf) \
(buf).write((s), strlen(s))
/** Пишет строку для использования в формате JSON:
* - строка выводится в двойных кавычках
* - эскейпится символ прямого слеша '/'
* - байты из диапазона 0x00-0x1F кроме '\b', '\f', '\n', '\r', '\t' эскейпятся как \u00XX
* - кодовые точки U+2028 и U+2029 (последовательности байт в UTF-8: e2 80 a8, e2 80 a9) эскейпятся как \u2028 и \u2029
* - предполагается, что строка в кодировке UTF-8, невалидный UTF-8 не обрабатывается
* - не-ASCII символы остаются как есть
*/
2013-05-16 12:52:09 +00:00
inline void writeJSONString(const char * begin, const char * end, WriteBuffer & buf)
{
writeChar('"', buf);
for (const char * it = begin; it != end; ++it)
{
switch (*it)
{
case '\b':
writeChar('\\', buf);
writeChar('b', buf);
break;
case '\f':
writeChar('\\', buf);
writeChar('f', buf);
break;
case '\n':
writeChar('\\', buf);
writeChar('n', buf);
break;
case '\r':
writeChar('\\', buf);
writeChar('r', buf);
break;
case '\t':
writeChar('\\', buf);
writeChar('t', buf);
2014-08-14 01:37:06 +00:00
break;
2013-05-16 12:52:09 +00:00
case '\\':
writeChar('\\', buf);
writeChar('\\', buf);
break;
case '/':
writeChar('\\', buf);
writeChar('/', buf);
break;
2013-09-02 15:32:17 +00:00
case '"':
writeChar('\\', buf);
writeChar('"', buf);
break;
2013-05-16 12:52:09 +00:00
default:
if (0x00 <= *it && *it <= 0x1F)
{
2013-05-20 09:33:45 +00:00
char higher_half = (*it) >> 4;
char lower_half = (*it) & 0xF;
2014-08-14 01:37:06 +00:00
2013-05-20 09:33:45 +00:00
writeCString("\\u00", buf);
writeChar('0' + higher_half, buf);
2014-08-14 01:37:06 +00:00
2013-05-20 09:33:45 +00:00
if (0 <= lower_half && lower_half <= 9)
writeChar('0' + lower_half, buf);
else
writeChar('A' + lower_half - 10, buf);
2013-05-16 12:52:09 +00:00
}
2013-05-20 09:33:45 +00:00
else if (end - it >= 3 && it[0] == '\xE2' && it[1] == '\x80' && (it[2] == '\xA8' || it[2] == '\xA9'))
2013-05-16 12:52:09 +00:00
{
2013-05-20 09:33:45 +00:00
if (it[2] == '\xA8')
writeCString("\\u2028", buf);
if (it[2] == '\xA9')
writeCString("\\u2029", buf);
2013-05-16 12:52:09 +00:00
}
else
writeChar(*it, buf);
}
}
writeChar('"', buf);
}
2011-11-01 17:57:37 +00:00
template <char c>
void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & buf)
2011-11-01 17:57:37 +00:00
{
for (const char * it = begin; it != end; ++it)
2011-11-01 17:57:37 +00:00
{
switch (*it)
{
case '\b':
writeChar('\\', buf);
writeChar('b', buf);
break;
case '\f':
writeChar('\\', buf);
writeChar('f', buf);
break;
case '\n':
writeChar('\\', buf);
writeChar('n', buf);
break;
case '\r':
writeChar('\\', buf);
writeChar('r', buf);
break;
case '\t':
writeChar('\\', buf);
writeChar('t', buf);
break;
case '\0':
writeChar('\\', buf);
writeChar('0', buf);
break;
case '\\':
writeChar('\\', buf);
writeChar('\\', buf);
break;
case c:
writeChar('\\', buf);
writeChar(c, buf);
break;
default:
writeChar(*it, buf);
}
}
}
2013-05-16 12:52:09 +00:00
inline void writeJSONString(const String & s, WriteBuffer & buf)
{
writeJSONString(s.data(), s.data() + s.size(), buf);
}
template <char c>
void writeAnyEscapedString(const String & s, WriteBuffer & buf)
{
writeAnyEscapedString<c>(s.data(), s.data() + s.size(), buf);
}
2011-11-01 17:57:37 +00:00
inline void writeEscapedString(const String & s, WriteBuffer & buf)
{
/// strpbrk в libc под Linux на процессорах с SSE 4.2 хорошо оптимизирована (этот if ускоряет код в 1.5 раза)
2014-04-08 07:47:51 +00:00
if (nullptr == strpbrk(s.data(), "\b\f\n\r\t\'\\") && strlen(s.data()) == s.size())
writeString(s, buf);
else
writeAnyEscapedString<'\''>(s, buf);
2011-11-01 17:57:37 +00:00
}
template <char c>
2013-04-15 19:33:05 +00:00
void writeAnyQuotedString(const char * begin, const char * end, WriteBuffer & buf)
2011-11-01 17:57:37 +00:00
{
writeChar(c, buf);
2013-04-15 19:33:05 +00:00
writeAnyEscapedString<c>(begin, end, buf);
2011-11-01 17:57:37 +00:00
writeChar(c, buf);
}
2010-06-04 18:25:25 +00:00
2013-04-15 19:33:05 +00:00
template <char c>
void writeAnyQuotedString(const String & s, WriteBuffer & buf)
{
writeAnyQuotedString<c>(s.data(), s.data() + s.size(), buf);
}
2010-06-04 18:25:25 +00:00
inline void writeQuotedString(const String & s, WriteBuffer & buf)
{
2011-11-01 17:57:37 +00:00
writeAnyQuotedString<'\''>(s, buf);
}
inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf)
{
writeAnyQuotedString<'"'>(s, buf);
2010-06-04 18:25:25 +00:00
}
2011-11-06 20:47:07 +00:00
/// Выводит строку в обратных кавычках, как идентификатор в MySQL.
2011-11-01 17:57:37 +00:00
inline void writeBackQuotedString(const String & s, WriteBuffer & buf)
{
writeAnyQuotedString<'`'>(s, buf);
}
2011-06-15 18:54:18 +00:00
2011-11-06 20:47:07 +00:00
/// То же самое, но обратные кавычки применяются только при наличии символов, не подходящих для идентификатора без обратных кавычек.
inline void writeProbablyBackQuotedString(const String & s, WriteBuffer & buf)
{
if (s.empty() || !((s[0] >= 'a' && s[0] <= 'z') || (s[0] >= 'A' && s[0] <= 'Z') || s[0] == '_'))
writeBackQuotedString(s, buf);
else
{
const char * pos = s.data() + 1;
const char * end = s.data() + s.size();
for (; pos < end; ++pos)
if (!((*pos >= 'a' && *pos <= 'z') || (*pos >= 'A' && *pos <= 'Z') || (*pos >= '0' && *pos <= '9') || *pos == '_'))
break;
if (pos != end)
writeBackQuotedString(s, buf);
else
writeString(s, buf);
}
}
2010-06-04 18:25:25 +00:00
2011-05-05 19:10:17 +00:00
/// в формате YYYY-MM-DD
inline void writeDateText(DayNum_t date, WriteBuffer & buf)
2011-05-05 19:10:17 +00:00
{
2011-08-07 03:42:36 +00:00
char s[10] = {'0', '0', '0', '0', '-', '0', '0', '-', '0', '0'};
if (unlikely(date > DATE_LUT_MAX_DAY_NUM || date == 0))
{
buf.write(s, 10);
return;
}
2011-05-05 19:10:17 +00:00
2014-07-08 23:52:53 +00:00
const DateLUT::Values & values = DateLUT::instance().getValues(date);
2011-05-05 19:10:17 +00:00
2011-08-07 03:42:36 +00:00
s[0] += values.year / 1000;
s[1] += (values.year / 100) % 10;
s[2] += (values.year / 10) % 10;
s[3] += values.year % 10;
s[5] += values.month / 10;
s[6] += values.month % 10;
s[8] += values.day_of_month / 10;
s[9] += values.day_of_month % 10;
2014-08-14 01:37:06 +00:00
2011-05-05 19:10:17 +00:00
buf.write(s, 10);
}
inline void writeDateText(mysqlxx::Date date, WriteBuffer & buf)
{
char s[10] = {'0', '0', '0', '0', '-', '0', '0', '-', '0', '0'};
s[0] += date.year() / 1000;
s[1] += (date.year() / 100) % 10;
s[2] += (date.year() / 10) % 10;
s[3] += date.year() % 10;
s[5] += date.month() / 10;
s[6] += date.month() % 10;
s[8] += date.day() / 10;
s[9] += date.day() % 10;
buf.write(s, 10);
}
2011-05-05 19:10:17 +00:00
2011-08-07 02:08:22 +00:00
/// в формате YYYY-MM-DD HH:MM:SS, согласно текущему часовому поясу
inline void writeDateTimeText(time_t datetime, WriteBuffer & buf)
{
2011-08-07 03:42:36 +00:00
char s[19] = {'0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0'};
if (unlikely(datetime > DATE_LUT_MAX || datetime == 0))
{
buf.write(s, 19);
return;
}
2011-08-07 02:08:22 +00:00
2014-07-08 23:52:53 +00:00
DateLUT & date_lut = DateLUT::instance();
const DateLUT::Values & values = date_lut.getValues(datetime);
2011-08-07 02:08:22 +00:00
2011-08-07 03:42:36 +00:00
s[0] += values.year / 1000;
s[1] += (values.year / 100) % 10;
s[2] += (values.year / 10) % 10;
s[3] += values.year % 10;
s[5] += values.month / 10;
s[6] += values.month % 10;
s[8] += values.day_of_month / 10;
s[9] += values.day_of_month % 10;
2011-08-07 02:08:22 +00:00
UInt8 hour = date_lut.toHourInaccurate(datetime);
2012-08-31 20:38:05 +00:00
UInt8 minute = date_lut.toMinuteInaccurate(datetime);
UInt8 second = date_lut.toSecondInaccurate(datetime);
2011-08-07 02:08:22 +00:00
2011-08-07 03:42:36 +00:00
s[11] += hour / 10;
s[12] += hour % 10;
s[14] += minute / 10;
s[15] += minute % 10;
s[17] += second / 10;
s[18] += second % 10;
2011-08-07 02:08:22 +00:00
buf.write(s, 19);
}
inline void writeDateTimeText(mysqlxx::DateTime datetime, WriteBuffer & buf)
{
char s[19] = {'0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0'};
s[0] += datetime.year() / 1000;
s[1] += (datetime.year() / 100) % 10;
s[2] += (datetime.year() / 10) % 10;
s[3] += datetime.year() % 10;
s[5] += datetime.month() / 10;
s[6] += datetime.month() % 10;
s[8] += datetime.day() / 10;
s[9] += datetime.day() % 10;
s[11] += datetime.hour() / 10;
s[12] += datetime.hour() % 10;
s[14] += datetime.minute() / 10;
s[15] += datetime.minute() % 10;
s[17] += datetime.second() / 10;
s[18] += datetime.second() % 10;
buf.write(s, 19);
}
/// Вывести mysqlxx::Row в tab-separated виде
inline void writeEscapedRow(const mysqlxx::Row & row, WriteBuffer & buf)
{
for (size_t i = 0; i < row.size(); ++i)
{
if (i != 0)
buf.write('\t');
if (unlikely(row[i].isNull()))
{
buf.write("\\N", 2);
continue;
}
writeAnyEscapedString<'\''>(row[i].data(), row[i].data() + row[i].length(), buf);
}
}
/// Методы вывода в бинарном виде
inline void writeBinary(const UInt8 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt16 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Int8 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Int16 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Int32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Int64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Float32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Float64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const bool & x, WriteBuffer & buf) { writePODBinary(x, buf); }
2014-04-02 13:45:39 +00:00
inline void writeBinary(const uint128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const VisitID_t & x, WriteBuffer & buf) { writePODBinary(static_cast<const UInt64 &>(x), buf); }
inline void writeBinary(const mysqlxx::Date & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const mysqlxx::DateTime & x, WriteBuffer & buf) { writePODBinary(x, buf); }
/// Методы для вывода значения в текстовом виде для tab-separated формата.
inline void writeText(const UInt8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const UInt16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const UInt32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const UInt64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const Int8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const Int16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const Int32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const Int64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeText(const Float32 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeText(const Float64 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeText(const String & x, WriteBuffer & buf) { writeEscapedString(x, buf); }
inline void writeText(const bool & x, WriteBuffer & buf) { writeBoolText(x, buf); }
inline void writeText(const VisitID_t & x, WriteBuffer & buf) { writeIntText(static_cast<const UInt64 &>(x), buf); }
inline void writeText(const mysqlxx::Date & x, WriteBuffer & buf) { writeDateText(x, buf); }
inline void writeText(const mysqlxx::DateTime & x, WriteBuffer & buf) { writeDateTimeText(x, buf); }
/// Методы для вывода в текстовом виде в кавычках
inline void writeQuoted(const UInt8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const UInt16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const UInt32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const UInt64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const Int8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const Int16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const Int32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const Int64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeQuoted(const Float32 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeQuoted(const Float64 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeQuoted(const String & x, WriteBuffer & buf) { writeQuotedString(x, buf); }
inline void writeQuoted(const bool & x, WriteBuffer & buf) { writeBoolText(x, buf); }
inline void writeQuoted(const VisitID_t & x, WriteBuffer & buf)
{
writeIntText(static_cast<const UInt64 &>(x), buf);
}
inline void writeQuoted(const mysqlxx::Date & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeDateText(x, buf);
writeChar('\'', buf);
}
inline void writeQuoted(const mysqlxx::DateTime & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeDateTimeText(x, buf);
writeChar('\'', buf);
}
/// В двойных кавычках
inline void writeDoubleQuoted(const UInt8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const UInt16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const UInt32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const UInt64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const Int8 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const Int16 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const Int32 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const Int64 & x, WriteBuffer & buf) { writeIntText(x, buf); }
inline void writeDoubleQuoted(const Float32 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeDoubleQuoted(const Float64 & x, WriteBuffer & buf) { writeFloatText(x, buf); }
inline void writeDoubleQuoted(const String & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); }
inline void writeDoubleQuoted(const bool & x, WriteBuffer & buf) { writeBoolText(x, buf); }
inline void writeDoubleQuoted(const VisitID_t & x, WriteBuffer & buf)
{
writeIntText(static_cast<const UInt64 &>(x), buf);
}
inline void writeDoubleQuoted(const mysqlxx::Date & x, WriteBuffer & buf)
{
writeChar('"', buf);
writeDateText(x, buf);
writeChar('"', buf);
}
inline void writeDoubleQuoted(const mysqlxx::DateTime & x, WriteBuffer & buf)
{
writeChar('"', buf);
writeDateTimeText(x, buf);
writeChar('"', buf);
}
template <typename T>
void writeBinary(const std::vector<T> & x, WriteBuffer & buf)
{
size_t size = x.size();
writeVarUInt(size, buf);
for (size_t i = 0; i < size; ++i)
writeBinary(x[i], buf);
}
template <typename T>
void writeQuoted(const std::vector<T> & x, WriteBuffer & buf)
{
writeChar('[', buf);
for (size_t i = 0, size = x.size(); i < size; ++i)
{
if (i != 0)
writeChar(',', buf);
writeQuoted(x[i], buf);
}
writeChar(']', buf);
}
template <typename T>
void writeDoubleQuoted(const std::vector<T> & x, WriteBuffer & buf)
{
writeChar('[', buf);
for (size_t i = 0, size = x.size(); i < size; ++i)
{
if (i != 0)
writeChar(',', buf);
writeDoubleQuoted(x[i], buf);
}
writeChar(']', buf);
}
template <typename T>
void writeText(const std::vector<T> & x, WriteBuffer & buf)
{
writeQuoted(x, buf);
}
2012-05-08 05:42:05 +00:00
/// Сериализация эксепшена (чтобы его можно было передать по сети)
void writeException(const Exception & e, WriteBuffer & buf);
2013-06-21 20:34:19 +00:00
/// Простой для использования метод преобразования чего-либо в строку в текстовом виде.
template <typename T>
inline String toString(const T & x)
{
String res;
{
WriteBufferFromString buf(res);
writeText(x, buf);
}
return res;
}
2010-06-04 18:25:25 +00:00
}