thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3669 from Enmk/ipv4_and_ipv6_domains

Browse Source 
Ipv4 and ipv6 domains
This commit is contained in:
KochetovNicolai 2019-02-06 18:16:58 +03:00 committed by GitHub
commit 9f971c276f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
64 changed files with 1515 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dbms/programs/copier/ClusterCopier.cpp
View File

@ -817,7 +817,7 @@ public:
try
{
type->deserializeTextQuoted(*column_dummy, rb, FormatSettings());
type->deserializeAsTextQuoted(*column_dummy, rb, FormatSettings());
}
catch (Exception & e)
{
@ -1882,7 +1882,7 @@ protected:
for (size_t i = 0; i < column.column->size(); ++i)
{
WriteBufferFromOwnString wb;
column.type->serializeTextQuoted(*column.column, i, wb, FormatSettings());
column.type->serializeAsTextQuoted(*column.column, i, wb, FormatSettings());
res.emplace(wb.str());
}
}

2
dbms/src/Columns/ColumnFixedString.h
View File

@ -138,7 +138,7 @@ public:
StringRef getRawData() const override { return StringRef(chars.data(), chars.size()); }
/// Specialized part of interface, not from IColumn.
void insertString(const String & string) { insertData(string.c_str(), string.size()); }
Chars & getChars() { return chars; }
const Chars & getChars() const { return chars; }

1
dbms/src/Common/ErrorCodes.cpp
View File

@ -414,6 +414,7 @@ namespace ErrorCodes
extern const int PROTOBUF_FIELD_NOT_REPEATED = 437;
extern const int DATA_TYPE_CANNOT_BE_PROMOTED = 438;
extern const int CANNOT_SCHEDULE_TASK = 439;
extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING = 440;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

60
dbms/src/Common/formatIPv6.cpp
View File

@ -1,12 +1,44 @@
#include <Common/formatIPv6.h>
#include <Common/hex.h>
#include <Common/StringUtils/StringUtils.h>
#include <ext/range.h>
#include <array>
#include <algorithm>
namespace DB
{
// To be used in formatIPv4, maps a byte to it's string form prefixed with length (so save strlen call).
const char one_byte_to_string_lookup_table[256][4] = {
{1, '0'}, {1, '1'}, {1, '2'}, {1, '3'}, {1, '4'}, {1, '5'}, {1, '6'}, {1, '7'}, {1, '8'}, {1, '9'},
{2, '1', '0'}, {2, '1', '1'}, {2, '1', '2'}, {2, '1', '3'}, {2, '1', '4'}, {2, '1', '5'}, {2, '1', '6'}, {2, '1', '7'}, {2, '1', '8'}, {2, '1', '9'},
{2, '2', '0'}, {2, '2', '1'}, {2, '2', '2'}, {2, '2', '3'}, {2, '2', '4'}, {2, '2', '5'}, {2, '2', '6'}, {2, '2', '7'}, {2, '2', '8'}, {2, '2', '9'},
{2, '3', '0'}, {2, '3', '1'}, {2, '3', '2'}, {2, '3', '3'}, {2, '3', '4'}, {2, '3', '5'}, {2, '3', '6'}, {2, '3', '7'}, {2, '3', '8'}, {2, '3', '9'},
{2, '4', '0'}, {2, '4', '1'}, {2, '4', '2'}, {2, '4', '3'}, {2, '4', '4'}, {2, '4', '5'}, {2, '4', '6'}, {2, '4', '7'}, {2, '4', '8'}, {2, '4', '9'},
{2, '5', '0'}, {2, '5', '1'}, {2, '5', '2'}, {2, '5', '3'}, {2, '5', '4'}, {2, '5', '5'}, {2, '5', '6'}, {2, '5', '7'}, {2, '5', '8'}, {2, '5', '9'},
{2, '6', '0'}, {2, '6', '1'}, {2, '6', '2'}, {2, '6', '3'}, {2, '6', '4'}, {2, '6', '5'}, {2, '6', '6'}, {2, '6', '7'}, {2, '6', '8'}, {2, '6', '9'},
{2, '7', '0'}, {2, '7', '1'}, {2, '7', '2'}, {2, '7', '3'}, {2, '7', '4'}, {2, '7', '5'}, {2, '7', '6'}, {2, '7', '7'}, {2, '7', '8'}, {2, '7', '9'},
{2, '8', '0'}, {2, '8', '1'}, {2, '8', '2'}, {2, '8', '3'}, {2, '8', '4'}, {2, '8', '5'}, {2, '8', '6'}, {2, '8', '7'}, {2, '8', '8'}, {2, '8', '9'},
{2, '9', '0'}, {2, '9', '1'}, {2, '9', '2'}, {2, '9', '3'}, {2, '9', '4'}, {2, '9', '5'}, {2, '9', '6'}, {2, '9', '7'}, {2, '9', '8'}, {2, '9', '9'},
{3, '1', '0', '0'}, {3, '1', '0', '1'}, {3, '1', '0', '2'}, {3, '1', '0', '3'}, {3, '1', '0', '4'}, {3, '1', '0', '5'}, {3, '1', '0', '6'}, {3, '1', '0', '7'}, {3, '1', '0', '8'}, {3, '1', '0', '9'},
{3, '1', '1', '0'}, {3, '1', '1', '1'}, {3, '1', '1', '2'}, {3, '1', '1', '3'}, {3, '1', '1', '4'}, {3, '1', '1', '5'}, {3, '1', '1', '6'}, {3, '1', '1', '7'}, {3, '1', '1', '8'}, {3, '1', '1', '9'},
{3, '1', '2', '0'}, {3, '1', '2', '1'}, {3, '1', '2', '2'}, {3, '1', '2', '3'}, {3, '1', '2', '4'}, {3, '1', '2', '5'}, {3, '1', '2', '6'}, {3, '1', '2', '7'}, {3, '1', '2', '8'}, {3, '1', '2', '9'},
{3, '1', '3', '0'}, {3, '1', '3', '1'}, {3, '1', '3', '2'}, {3, '1', '3', '3'}, {3, '1', '3', '4'}, {3, '1', '3', '5'}, {3, '1', '3', '6'}, {3, '1', '3', '7'}, {3, '1', '3', '8'}, {3, '1', '3', '9'},
{3, '1', '4', '0'}, {3, '1', '4', '1'}, {3, '1', '4', '2'}, {3, '1', '4', '3'}, {3, '1', '4', '4'}, {3, '1', '4', '5'}, {3, '1', '4', '6'}, {3, '1', '4', '7'}, {3, '1', '4', '8'}, {3, '1', '4', '9'},
{3, '1', '5', '0'}, {3, '1', '5', '1'}, {3, '1', '5', '2'}, {3, '1', '5', '3'}, {3, '1', '5', '4'}, {3, '1', '5', '5'}, {3, '1', '5', '6'}, {3, '1', '5', '7'}, {3, '1', '5', '8'}, {3, '1', '5', '9'},
{3, '1', '6', '0'}, {3, '1', '6', '1'}, {3, '1', '6', '2'}, {3, '1', '6', '3'}, {3, '1', '6', '4'}, {3, '1', '6', '5'}, {3, '1', '6', '6'}, {3, '1', '6', '7'}, {3, '1', '6', '8'}, {3, '1', '6', '9'},
{3, '1', '7', '0'}, {3, '1', '7', '1'}, {3, '1', '7', '2'}, {3, '1', '7', '3'}, {3, '1', '7', '4'}, {3, '1', '7', '5'}, {3, '1', '7', '6'}, {3, '1', '7', '7'}, {3, '1', '7', '8'}, {3, '1', '7', '9'},
{3, '1', '8', '0'}, {3, '1', '8', '1'}, {3, '1', '8', '2'}, {3, '1', '8', '3'}, {3, '1', '8', '4'}, {3, '1', '8', '5'}, {3, '1', '8', '6'}, {3, '1', '8', '7'}, {3, '1', '8', '8'}, {3, '1', '8', '9'},
{3, '1', '9', '0'}, {3, '1', '9', '1'}, {3, '1', '9', '2'}, {3, '1', '9', '3'}, {3, '1', '9', '4'}, {3, '1', '9', '5'}, {3, '1', '9', '6'}, {3, '1', '9', '7'}, {3, '1', '9', '8'}, {3, '1', '9', '9'},
{3, '2', '0', '0'}, {3, '2', '0', '1'}, {3, '2', '0', '2'}, {3, '2', '0', '3'}, {3, '2', '0', '4'}, {3, '2', '0', '5'}, {3, '2', '0', '6'}, {3, '2', '0', '7'}, {3, '2', '0', '8'}, {3, '2', '0', '9'},
{3, '2', '1', '0'}, {3, '2', '1', '1'}, {3, '2', '1', '2'}, {3, '2', '1', '3'}, {3, '2', '1', '4'}, {3, '2', '1', '5'}, {3, '2', '1', '6'}, {3, '2', '1', '7'}, {3, '2', '1', '8'}, {3, '2', '1', '9'},
{3, '2', '2', '0'}, {3, '2', '2', '1'}, {3, '2', '2', '2'}, {3, '2', '2', '3'}, {3, '2', '2', '4'}, {3, '2', '2', '5'}, {3, '2', '2', '6'}, {3, '2', '2', '7'}, {3, '2', '2', '8'}, {3, '2', '2', '9'},
{3, '2', '3', '0'}, {3, '2', '3', '1'}, {3, '2', '3', '2'}, {3, '2', '3', '3'}, {3, '2', '3', '4'}, {3, '2', '3', '5'}, {3, '2', '3', '6'}, {3, '2', '3', '7'}, {3, '2', '3', '8'}, {3, '2', '3', '9'},
{3, '2', '4', '0'}, {3, '2', '4', '1'}, {3, '2', '4', '2'}, {3, '2', '4', '3'}, {3, '2', '4', '4'}, {3, '2', '4', '5'}, {3, '2', '4', '6'}, {3, '2', '4', '7'}, {3, '2', '4', '8'}, {3, '2', '4', '9'},
{3, '2', '5', '0'}, {3, '2', '5', '1'}, {3, '2', '5', '2'}, {3, '2', '5', '3'}, {3, '2', '5', '4'}, {3, '2', '5', '5'},
};
/// integer logarithm, return ceil(log(value, base)) (the smallest integer greater or equal than log(value, base)
static constexpr UInt32 intLog(const UInt32 value, const UInt32 base, const bool carry)
{
@ -45,22 +77,6 @@ static void printInteger(char *& out, T value)
}
}
/// print IPv4 address as %u.%u.%u.%u
static void formatIPv4(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count)
{
const auto limit = IPV4_BINARY_LENGTH - zeroed_tail_bytes_count;
for (const auto i : ext::range(0, IPV4_BINARY_LENGTH))
{
UInt8 byte = (i < limit) ? src[i] : 0;
printInteger<10, UInt8>(dst, byte);
if (i != IPV4_BINARY_LENGTH - 1)
*dst++ = '.';
}
}
void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count)
{
struct { int base, len; } best{-1, 0}, cur{-1, 0};
@ -122,8 +138,14 @@ void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_
/// Is this address an encapsulated IPv4?
if (i == 6 && best.base == 0 && (best.len == 6 || (best.len == 5 && words[5] == 0xffffu)))
{
formatIPv4(src + 12, dst, std::min(zeroed_tail_bytes_count, static_cast<UInt8>(IPV4_BINARY_LENGTH)));
break;
UInt8 ipv4_buffer[IPV4_BINARY_LENGTH] = {0};
memcpy(ipv4_buffer, src + 12, IPV4_BINARY_LENGTH);
// Due to historical reasons formatIPv4() takes ipv4 in BE format, but inside ipv6 we store it in LE-format.
std::reverse(std::begin(ipv4_buffer), std::end(ipv4_buffer));
formatIPv4(ipv4_buffer, dst, std::min(zeroed_tail_bytes_count, static_cast<UInt8>(IPV4_BINARY_LENGTH)), "0");
// formatIPv4 has already added a null-terminator for us.
return;
}
printInteger<16>(dst, words[i]);

216
dbms/src/Common/formatIPv6.h
View File

@ -1,12 +1,17 @@
#pragma once
#include <common/Types.h>
#include <string.h>
#include <algorithm>
#include <utility>
#include <ext/range.h>
#include <Common/hex.h>
#include <Common/StringUtils/StringUtils.h>
#define IPV4_BINARY_LENGTH 4
#define IPV6_BINARY_LENGTH 16
#define IPV4_MAX_TEXT_LENGTH 15 /// Does not count tail zero byte.
#define IPV6_MAX_TEXT_LENGTH 39
constexpr size_t IPV4_BINARY_LENGTH = 4;
constexpr size_t IPV6_BINARY_LENGTH = 16;
constexpr size_t IPV4_MAX_TEXT_LENGTH = 15; /// Does not count tail zero byte.
constexpr size_t IPV6_MAX_TEXT_LENGTH = 39;
namespace DB
{
@ -18,4 +23,205 @@ namespace DB
*/
void formatIPv6(const unsigned char * src, char *& dst, UInt8 zeroed_tail_bytes_count = 0);
/** Unsafe (no bounds-checking for src nor dst), optimized version of parsing IPv4 string.
*
* Parses the input string `src` and stores binary BE value into buffer pointed by `dst`,
* which should be long enough.
* That is "127.0.0.1" becomes 0x7f000001.
*
* In case of failure returns false and doesn't modify buffer pointed by `dst`.
*
* @param src - input string, expected to be non-null and null-terminated right after the IPv4 string value.
* @param dst - where to put output bytes, expected to be non-null and atleast IPV4_BINARY_LENGTH-long.
* @return false if parsing failed, true otherwise.
*/
inline bool parseIPv4(const char * src, unsigned char * dst)
{
UInt32 result = 0;
for (int offset = 24; offset >= 0; offset -= 8)
{
UInt32 value = 0;
size_t len = 0;
while (isNumericASCII(*src) && len <= 3)
{
value = value * 10 + (*src - '0');
++len;
++src;
}
if (len == 0 || value > 255 || (offset > 0 && *src != '.'))
return false;
result |= value << offset;
++src;
}
if (*(src - 1) != '\0')
return false;
memcpy(dst, &result, sizeof(result));
return true;
}
/** Unsafe (no bounds-checking for src nor dst), optimized version of parsing IPv6 string.
*
* Slightly altered implementation from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
* Parses the input string `src` and stores binary LE value into buffer pointed by `dst`,
* which should be long enough. In case of failure zeroes
* IPV6_BINARY_LENGTH bytes of buffer pointed by `dst`.
*
* @param src - input string, expected to be non-null and null-terminated right after the IPv6 string value.
* @param dst - where to put output bytes, expected to be non-null and atleast IPV6_BINARY_LENGTH-long.
* @return false if parsing failed, true otherwise.
*/
inline bool parseIPv6(const char * src, unsigned char * dst)
{
const auto clear_dst = [dst]()
{
memset(dst, '\0', IPV6_BINARY_LENGTH);
return false;
};
/// Leading :: requires some special handling.
if (*src == ':')
if (*++src != ':')
return clear_dst();
unsigned char tmp[IPV6_BINARY_LENGTH]{};
auto tp = tmp;
auto endp = tp + IPV6_BINARY_LENGTH;
auto curtok = src;
auto saw_xdigit = false;
UInt32 val{};
unsigned char * colonp = nullptr;
/// Assuming zero-terminated string.
while (const auto ch = *src++)
{
const auto num = unhex(ch);
if (num != '\xff')
{
val <<= 4;
val |= num;
if (val > 0xffffu)
return clear_dst();
saw_xdigit = 1;
continue;
}
if (ch == ':')
{
curtok = src;
if (!saw_xdigit)
{
if (colonp)
return clear_dst();
colonp = tp;
continue;
}
if (tp + sizeof(UInt16) > endp)
return clear_dst();
*tp++ = static_cast<unsigned char>((val >> 8) & 0xffu);
*tp++ = static_cast<unsigned char>(val & 0xffu);
saw_xdigit = false;
val = 0;
continue;
}
if (ch == '.' && (tp + IPV4_BINARY_LENGTH) <= endp)
{
if (!parseIPv4(curtok, tp))
return clear_dst();
std::reverse(tp, tp + IPV4_BINARY_LENGTH);
tp += IPV4_BINARY_LENGTH;
saw_xdigit = false;
break; /* '\0' was seen by ipv4_scan(). */
}
return clear_dst();
}
if (saw_xdigit)
{
if (tp + sizeof(UInt16) > endp)
return clear_dst();
*tp++ = static_cast<unsigned char>((val >> 8) & 0xffu);
*tp++ = static_cast<unsigned char>(val & 0xffu);
}
if (colonp)
{
/*
* Since some memmove()'s erroneously fail to handle
* overlapping regions, we'll do the shift by hand.
*/
const auto n = tp - colonp;
for (int i = 1; i <= n; ++i)
{
endp[- i] = colonp[n - i];
colonp[n - i] = 0;
}
tp = endp;
}
if (tp != endp)
return clear_dst();
memcpy(dst, tmp, sizeof(tmp));
return true;
}
/** Format 4-byte binary sequesnce as IPv4 text: 'aaa.bbb.ccc.ddd',
* expects inout to be in BE-format, that is 0x7f000001 => "127.0.0.1".
*
* Any number of the tail bytes can be masked with given mask string.
*
* Assumptions:
* src is IPV4_BINARY_LENGTH long,
* dst is IPV4_MAX_TEXT_LENGTH long,
* mask_tail_octets <= IPV4_BINARY_LENGTH
* mask_string is NON-NULL, if mask_tail_octets > 0.
*
* Examples:
* formatIPv4(&0x7f000001, dst, mask_tail_octets = 0, nullptr);
* > dst == "127.0.0.1"
* formatIPv4(&0x7f000001, dst, mask_tail_octets = 1, "xxx");
* > dst == "127.0.0.xxx"
* formatIPv4(&0x7f000001, dst, mask_tail_octets = 1, "0");
* > dst == "127.0.0.0"
*/
inline void formatIPv4(const unsigned char * src, char *& dst, UInt8 mask_tail_octets = 0, const char * mask_string = "xxx")
{
extern const char one_byte_to_string_lookup_table[256][4];
const size_t mask_length = mask_string ? strlen(mask_string) : 0;
const size_t limit = std::min(IPV4_BINARY_LENGTH, IPV4_BINARY_LENGTH - mask_tail_octets);
for (size_t octet = 0; octet < limit; ++octet)
{
const UInt8 value = static_cast<UInt8>(src[IPV4_BINARY_LENGTH - octet - 1]);
auto rep = one_byte_to_string_lookup_table[value];
const UInt8 len = rep[0];
const char* str = rep + 1;
memcpy(dst, str, len);
dst += len;
*dst++ = '.';
}
for (size_t mask = 0; mask < mask_tail_octets; ++mask)
{
memcpy(dst, mask_string, mask_length);
dst += mask_length;
*dst++ = '.';
}
dst[-1] = '\0';
}
}

2
dbms/src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -32,7 +32,7 @@ namespace ErrorCodes
}
std::string DataTypeAggregateFunction::getName() const
std::string DataTypeAggregateFunction::doGetName() const
{
std::stringstream stream;
stream << "AggregateFunction(" << function->getName();

2
dbms/src/DataTypes/DataTypeAggregateFunction.h
View File

@ -29,7 +29,7 @@ public:
std::string getFunctionName() const { return function->getName(); }
AggregateFunctionPtr getFunction() const { return function; }
std::string getName() const override;
std::string doGetName() const override;
const char * getFamilyName() const override { return "AggregateFunction"; }
TypeIndex getTypeId() const override { return TypeIndex::AggregateFunction; }

10
dbms/src/DataTypes/DataTypeArray.cpp
View File

@ -350,7 +350,7 @@ void DataTypeArray::serializeText(const IColumn & column, size_t row_num, WriteB
serializeTextImpl(column, row_num, ostr,
[&](const IColumn & nested_column, size_t i)
{
nested->serializeTextQuoted(nested_column, i, ostr, settings);
nested->serializeAsTextQuoted(nested_column, i, ostr, settings);
});
}
@ -360,7 +360,7 @@ void DataTypeArray::deserializeText(IColumn & column, ReadBuffer & istr, const F
deserializeTextImpl(column, istr,
[&](IColumn & nested_column)
{
nested->deserializeTextQuoted(nested_column, istr, settings);
nested->deserializeAsTextQuoted(nested_column, istr, settings);
});
}
@ -379,7 +379,7 @@ void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, Wr
{
if (i != offset)
writeChar(',', ostr);
nested->serializeTextJSON(nested_column, i, ostr, settings);
nested->serializeAsTextJSON(nested_column, i, ostr, settings);
}
writeChar(']', ostr);
}
@ -387,7 +387,7 @@ void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, Wr
void DataTypeArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeTextImpl(column, istr, [&](IColumn & nested_column) { nested->deserializeTextJSON(nested_column, istr, settings); });
deserializeTextImpl(column, istr, [&](IColumn & nested_column) { nested->deserializeAsTextJSON(nested_column, istr, settings); });
}
@ -405,7 +405,7 @@ void DataTypeArray::serializeTextXML(const IColumn & column, size_t row_num, Wri
for (size_t i = offset; i < next_offset; ++i)
{
writeCString("<elem>", ostr);
nested->serializeTextXML(nested_column, i, ostr, settings);
nested->serializeAsTextXML(nested_column, i, ostr, settings);
writeCString("</elem>", ostr);
}
writeCString("</array>", ostr);

2
dbms/src/DataTypes/DataTypeArray.h
View File

@ -20,7 +20,7 @@ public:
TypeIndex getTypeId() const override { return TypeIndex::Array; }
std::string getName() const override
std::string doGetName() const override
{
return "Array(" + nested->getName() + ")";
}

2
dbms/src/DataTypes/DataTypeDateTime.cpp
View File

@ -26,7 +26,7 @@ DataTypeDateTime::DataTypeDateTime(const std::string & time_zone_name)
{
}
std::string DataTypeDateTime::getName() const
std::string DataTypeDateTime::doGetName() const
{
if (!has_explicit_time_zone)
return "DateTime";

2
dbms/src/DataTypes/DataTypeDateTime.h
View File

@ -34,7 +34,7 @@ public:
DataTypeDateTime(const std::string & time_zone_name = "");
const char * getFamilyName() const override { return "DateTime"; }
std::string getName() const override;
std::string doGetName() const override;
TypeIndex getTypeId() const override { return TypeIndex::DateTime; }
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

118
dbms/src/DataTypes/DataTypeDomainIPv4AndIPv6.cpp Normal file
View File

@ -0,0 +1,118 @@
#include <Columns/ColumnsNumber.h>
#include <Common/Exception.h>
#include <Common/formatIPv6.h>
#include <DataTypes/DataTypeDomainWithSimpleSerialization.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/IDataTypeDomain.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/FunctionsCoding.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int UNSUPPORTED_METHOD;
extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING;
}
namespace
{
class DataTypeDomanIPv4 : public DataTypeDomainWithSimpleSerialization
{
public:
const char * getName() const override
{
return "IPv4";
}
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override
{
const auto col = checkAndGetColumn<ColumnUInt32>(&column);
if (!col)
{
throw Exception(String(getName()) + " domain can only serialize columns of type UInt32." + column.getName(), ErrorCodes::ILLEGAL_COLUMN);
}
char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'};
char * ptr = buffer;
formatIPv4(reinterpret_cast<const unsigned char *>(&col->getData()[row_num]), ptr);
ostr.write(buffer, strlen(buffer));
}
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override
{
ColumnUInt32 * col = typeid_cast<ColumnUInt32 *>(&column);
if (!col)
{
throw Exception(String(getName()) + " domain can only deserialize columns of type UInt32." + column.getName(), ErrorCodes::ILLEGAL_COLUMN);
}
char buffer[IPV4_MAX_TEXT_LENGTH + 1] = {'\0'};
istr.read(buffer, sizeof(buffer) - 1);
UInt32 ipv4_value = 0;
if (!parseIPv4(buffer, reinterpret_cast<unsigned char *>(&ipv4_value)))
{
throw Exception("Invalid IPv4 value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
}
col->insert(ipv4_value);
}
};
class DataTypeDomanIPv6 : public DataTypeDomainWithSimpleSerialization
{
public:
const char * getName() const override
{
return "IPv6";
}
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override
{
const auto col = checkAndGetColumn<ColumnFixedString>(&column);
if (!col)
{
throw Exception(String(getName()) + " domain can only serialize columns of type FixedString(16)." + column.getName(), ErrorCodes::ILLEGAL_COLUMN);
}
char buffer[IPV6_MAX_TEXT_LENGTH + 1] = {'\0'};
char * ptr = buffer;
formatIPv6(reinterpret_cast<const unsigned char *>(col->getDataAt(row_num).data), ptr);
ostr.write(buffer, strlen(buffer));
}
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override
{
ColumnFixedString * col = typeid_cast<ColumnFixedString *>(&column);
if (!col)
{
throw Exception(String(getName()) + " domain can only deserialize columns of type FixedString(16)." + column.getName(), ErrorCodes::ILLEGAL_COLUMN);
}
char buffer[IPV6_MAX_TEXT_LENGTH + 1] = {'\0'};
istr.read(buffer, sizeof(buffer) - 1);
std::string ipv6_value(IPV6_BINARY_LENGTH, '\0');
if (!parseIPv6(buffer, reinterpret_cast<unsigned char *>(ipv6_value.data())))
{
throw Exception(String("Invalid ") + getName() + " value.", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING);
}
col->insertString(ipv6_value);
}
};
} // namespace
void registerDataTypeDomainIPv4AndIPv6(DataTypeFactory & factory)
{
factory.registerDataTypeDomain("UInt32", std::make_unique<DataTypeDomanIPv4>());
factory.registerDataTypeDomain("FixedString(16)", std::make_unique<DataTypeDomanIPv6>());
}
} // namespace DB

88
dbms/src/DataTypes/DataTypeDomainWithSimpleSerialization.cpp Normal file
View File

@ -0,0 +1,88 @@
#include <DataTypes/DataTypeDomainWithSimpleSerialization.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
namespace
{
using namespace DB;
static String serializeToString(const DataTypeDomainWithSimpleSerialization & domain, const IColumn & column, size_t row_num, const FormatSettings & settings)
{
WriteBufferFromOwnString buffer;
domain.serializeText(column, row_num, buffer, settings);
return buffer.str();
}
static void deserializeFromString(const DataTypeDomainWithSimpleSerialization & domain, IColumn & column, const String & s, const FormatSettings & settings)
{
ReadBufferFromString istr(s);
domain.deserializeText(column, istr, settings);
}
} // namespace
namespace DB
{
DataTypeDomainWithSimpleSerialization::~DataTypeDomainWithSimpleSerialization()
{
}
void DataTypeDomainWithSimpleSerialization::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
writeEscapedString(serializeToString(*this, column, row_num, settings), ostr);
}
void DataTypeDomainWithSimpleSerialization::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String str;
readEscapedString(str, istr);
deserializeFromString(*this, column, str, settings);
}
void DataTypeDomainWithSimpleSerialization::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
writeQuotedString(serializeToString(*this, column, row_num, settings), ostr);
}
void DataTypeDomainWithSimpleSerialization::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String str;
readQuotedString(str, istr);
deserializeFromString(*this, column, str, settings);
}
void DataTypeDomainWithSimpleSerialization::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
writeCSVString(serializeToString(*this, column, row_num, settings), ostr);
}
void DataTypeDomainWithSimpleSerialization::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String str;
readCSVString(str, istr, settings.csv);
deserializeFromString(*this, column, str, settings);
}
void DataTypeDomainWithSimpleSerialization::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
writeJSONString(serializeToString(*this, column, row_num, settings), ostr, settings);
}
void DataTypeDomainWithSimpleSerialization::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String str;
readJSONString(str, istr);
deserializeFromString(*this, column, str, settings);
}
void DataTypeDomainWithSimpleSerialization::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
writeXMLString(serializeToString(*this, column, row_num, settings), ostr);
}
} // namespace DB

53
dbms/src/DataTypes/DataTypeDomainWithSimpleSerialization.h Normal file
View File

@ -0,0 +1,53 @@
#pragma once
#include <DataTypes/IDataTypeDomain.h>
namespace DB
{
class ReadBuffer;
class WriteBuffer;
struct FormatSettings;
class IColumn;
/** Simple DataTypeDomain that uses serializeText/deserializeText
* for all serialization and deserialization. */
class DataTypeDomainWithSimpleSerialization : public IDataTypeDomain
{
public:
virtual ~DataTypeDomainWithSimpleSerialization() override;
// Methods that subclasses must override in order to get full serialization/deserialization support.
virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override = 0;
virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization with escaping but without quoting.
*/
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
/** Text serialization as a literal that may be inserted into a query.
*/
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
/** Text serialization for the CSV format.
*/
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
/** delimiter - the delimiter we expect when reading a string value that is not double-quoted
* (the delimiter is not consumed).
*/
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
/** Text serialization intended for using in JSON format.
* force_quoting_64bit_integers parameter forces to brace UInt64 and Int64 types into quotes.
*/
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
/** Text serialization for putting into the XML format.
*/
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override;
};
} // namespace DB

2
dbms/src/DataTypes/DataTypeEnum.h
View File

@ -61,7 +61,7 @@ public:
explicit DataTypeEnum(const Values & values_);
const Values & getValues() const { return values; }
std::string getName() const override { return type_name; }
std::string doGetName() const override { return type_name; }
const char * getFamilyName() const override;
TypeIndex getTypeId() const override { return sizeof(FieldType) == 1 ? TypeIndex::Enum8 : TypeIndex::Enum16; }

69
dbms/src/DataTypes/DataTypeFactory.cpp
View File

@ -1,4 +1,5 @@
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/IDataTypeDomain.h>
#include <Parsers/parseQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ASTFunction.h>
@ -73,25 +74,7 @@ DataTypePtr DataTypeFactory::get(const String & family_name_param, const ASTPtr
return get("LowCardinality", low_cardinality_params);
}
{
DataTypesDictionary::const_iterator it = data_types.find(family_name);
if (data_types.end() != it)
return it->second(parameters);
}
String family_name_lowercase = Poco::toLower(family_name);
{
DataTypesDictionary::const_iterator it = case_insensitive_data_types.find(family_name_lowercase);
if (case_insensitive_data_types.end() != it)
return it->second(parameters);
}
auto hints = this->getHints(family_name);
if (!hints.empty())
throw Exception("Unknown data type family: " + family_name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_TYPE);
else
throw Exception("Unknown data type family: " + family_name, ErrorCodes::UNKNOWN_TYPE);
return findCreatorByName(family_name)(parameters);
}
@ -132,6 +115,49 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator
}, case_sensitiveness);
}
void DataTypeFactory::registerDataTypeDomain(const String & type_name, DataTypeDomainPtr domain, CaseSensitiveness case_sensitiveness)
{
all_domains.reserve(all_domains.size() + 1);
auto data_type = get(type_name);
setDataTypeDomain(*data_type, *domain);
registerDataType(domain->getName(), [data_type](const ASTPtr & /*ast*/)
{
return data_type;
}, case_sensitiveness);
all_domains.emplace_back(std::move(domain));
}
const DataTypeFactory::Creator& DataTypeFactory::findCreatorByName(const String & family_name) const
{
{
DataTypesDictionary::const_iterator it = data_types.find(family_name);
if (data_types.end() != it)
return it->second;
}
String family_name_lowercase = Poco::toLower(family_name);
{
DataTypesDictionary::const_iterator it = case_insensitive_data_types.find(family_name_lowercase);
if (case_insensitive_data_types.end() != it)
return it->second;
}
auto hints = this->getHints(family_name);
if (!hints.empty())
throw Exception("Unknown data type family: " + family_name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_TYPE);
else
throw Exception("Unknown data type family: " + family_name, ErrorCodes::UNKNOWN_TYPE);
}
void DataTypeFactory::setDataTypeDomain(const IDataType & data_type, const IDataTypeDomain & domain)
{
data_type.setDomain(&domain);
}
void registerDataTypeNumbers(DataTypeFactory & factory);
void registerDataTypeDecimal(DataTypeFactory & factory);
void registerDataTypeDate(DataTypeFactory & factory);
@ -148,6 +174,7 @@ void registerDataTypeAggregateFunction(DataTypeFactory & factory);
void registerDataTypeNested(DataTypeFactory & factory);
void registerDataTypeInterval(DataTypeFactory & factory);
void registerDataTypeLowCardinality(DataTypeFactory & factory);
void registerDataTypeDomainIPv4AndIPv6(DataTypeFactory & factory);
DataTypeFactory::DataTypeFactory()
@ -168,6 +195,10 @@ DataTypeFactory::DataTypeFactory()
registerDataTypeNested(*this);
registerDataTypeInterval(*this);
registerDataTypeLowCardinality(*this);
registerDataTypeDomainIPv4AndIPv6(*this);
}
DataTypeFactory::~DataTypeFactory()
{}
}

14
dbms/src/DataTypes/DataTypeFactory.h
View File

@ -14,6 +14,9 @@ namespace DB
class IDataType;
using DataTypePtr = std::shared_ptr<const IDataType>;
class IDataTypeDomain;
using DataTypeDomainPtr = std::unique_ptr<const IDataTypeDomain>;
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
@ -37,13 +40,24 @@ public:
/// Register a simple data type, that have no parameters.
void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
// Register a domain - a refinement of existing type.
void registerDataTypeDomain(const String & type_name, DataTypeDomainPtr domain, CaseSensitiveness case_sensitiveness = CaseSensitive);
private:
static void setDataTypeDomain(const IDataType & data_type, const IDataTypeDomain & domain);
const Creator& findCreatorByName(const String & family_name) const;
private:
DataTypesDictionary data_types;
/// Case insensitive data types will be additionally added here with lowercased name.
DataTypesDictionary case_insensitive_data_types;
// All domains are owned by factory and shared amongst DataType instances.
std::vector<DataTypeDomainPtr> all_domains;
DataTypeFactory();
~DataTypeFactory() override;
const DataTypesDictionary & getCreatorMap() const override { return data_types; }

2
dbms/src/DataTypes/DataTypeFixedString.cpp
View File

@ -32,7 +32,7 @@ namespace ErrorCodes
}
std::string DataTypeFixedString::getName() const
std::string DataTypeFixedString::doGetName() const
{
return "FixedString(" + toString(n) + ")";
}

2
dbms/src/DataTypes/DataTypeFixedString.h
View File

@ -30,7 +30,7 @@ public:
throw Exception("FixedString size is too large", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
}
std::string getName() const override;
std::string doGetName() const override;
TypeIndex getTypeId() const override { return TypeIndex::FixedString; }
const char * getFamilyName() const override { return "FixedString"; }

2
dbms/src/DataTypes/DataTypeFunction.cpp
View File

@ -6,7 +6,7 @@
namespace DB
{
std::string DataTypeFunction::getName() const
std::string DataTypeFunction::doGetName() const
{
WriteBufferFromOwnString res;

2
dbms/src/DataTypes/DataTypeFunction.h
View File

@ -22,7 +22,7 @@ public:
DataTypeFunction(const DataTypes & argument_types_ = DataTypes(), const DataTypePtr & return_type_ = nullptr)
: argument_types(argument_types_), return_type(return_type_) {}
std::string getName() const override;
std::string doGetName() const override;
const char * getFamilyName() const override { return "Function"; }
TypeIndex getTypeId() const override { return TypeIndex::Function; }

2
dbms/src/DataTypes/DataTypeInterval.h
View File

@ -56,7 +56,7 @@ public:
DataTypeInterval(Kind kind) : kind(kind) {}
std::string getName() const override { return std::string("Interval") + kindToString(); }
std::string doGetName() const override { return std::string("Interval") + kindToString(); }
const char * getFamilyName() const override { return "Interval"; }
TypeIndex getTypeId() const override { return TypeIndex::Interval; }

22
dbms/src/DataTypes/DataTypeLowCardinality.h
View File

@ -15,7 +15,7 @@ public:
const DataTypePtr & getDictionaryType() const { return dictionary_type; }
String getName() const override
String doGetName() const override
{
return "LowCardinality(" + dictionary_type->getName() + ")";
}
@ -63,51 +63,51 @@ public:
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeTextEscaped, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsTextEscaped, ostr, settings);
}
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeImpl(column, &IDataType::deserializeTextEscaped, istr, settings);
deserializeImpl(column, &IDataType::deserializeAsTextEscaped, istr, settings);
}
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeTextQuoted, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsTextQuoted, ostr, settings);
}
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeImpl(column, &IDataType::deserializeTextQuoted, istr, settings);
deserializeImpl(column, &IDataType::deserializeAsTextQuoted, istr, settings);
}
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeTextCSV, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsTextCSV, ostr, settings);
}
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeImpl(column, &IDataType::deserializeTextCSV, istr, settings);
deserializeImpl(column, &IDataType::deserializeAsTextCSV, istr, settings);
}
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeText, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsText, ostr, settings);
}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeTextJSON, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsTextJSON, ostr, settings);
}
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeImpl(column, &IDataType::deserializeTextJSON, istr, settings);
deserializeImpl(column, &IDataType::deserializeAsTextJSON, istr, settings);
}
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override
{
serializeImpl(column, row_num, &IDataType::serializeTextXML, ostr, settings);
serializeImpl(column, row_num, &IDataType::serializeAsTextXML, ostr, settings);
}
void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf) const override

24
dbms/src/DataTypes/DataTypeNullable.cpp
View File

@ -172,7 +172,7 @@ void DataTypeNullable::serializeTextEscaped(const IColumn & column, size_t row_n
if (col.isNullAt(row_num))
writeCString("\\N", ostr);
else
nested_data_type->serializeTextEscaped(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsTextEscaped(col.getNestedColumn(), row_num, ostr, settings);
}
@ -188,7 +188,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
{
safeDeserialize(column,
[] { return false; },
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextEscaped(nested, istr, settings); });
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextEscaped(nested, istr, settings); } );
}
else
{
@ -214,7 +214,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
{
/// We could step back to consume backslash again.
--istr.position();
nested_data_type->deserializeTextEscaped(nested, istr, settings);
nested_data_type->deserializeAsTextEscaped(nested, istr, settings);
}
else
{
@ -222,7 +222,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
ReadBufferFromMemory prefix("\\", 1);
ConcatReadBuffer prepended_istr(prefix, istr);
nested_data_type->deserializeTextEscaped(nested, prepended_istr, settings);
nested_data_type->deserializeAsTextEscaped(nested, prepended_istr, settings);
/// Synchronise cursor position in original buffer.
@ -240,7 +240,7 @@ void DataTypeNullable::serializeTextQuoted(const IColumn & column, size_t row_nu
if (col.isNullAt(row_num))
writeCString("NULL", ostr);
else
nested_data_type->serializeTextQuoted(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsTextQuoted(col.getNestedColumn(), row_num, ostr, settings);
}
@ -248,7 +248,7 @@ void DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr
{
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr, settings); });
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextQuoted(nested, istr, settings); });
}
void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@ -258,14 +258,14 @@ void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num,
if (col.isNullAt(row_num))
writeCString("\\N", ostr);
else
nested_data_type->serializeTextCSV(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsTextCSV(col.getNestedColumn(), row_num, ostr, settings);
}
void DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRest("\\N", istr); },
[this, &settings, &istr] (IColumn & nested) { nested_data_type->deserializeTextCSV(nested, istr, settings); });
[this, &settings, &istr] (IColumn & nested) { nested_data_type->deserializeAsTextCSV(nested, istr, settings); });
}
void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@ -281,7 +281,7 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri
if (col.isNullAt(row_num))
writeCString("ᴺᵁᴸᴸ", ostr);
else
nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsText(col.getNestedColumn(), row_num, ostr, settings);
}
void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@ -291,14 +291,14 @@ void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num,
if (col.isNullAt(row_num))
writeCString("null", ostr);
else
nested_data_type->serializeTextJSON(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsTextJSON(col.getNestedColumn(), row_num, ostr, settings);
}
void DataTypeNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRest("null", istr); },
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextJSON(nested, istr, settings); });
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextJSON(nested, istr, settings); });
}
void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@ -308,7 +308,7 @@ void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num,
if (col.isNullAt(row_num))
writeCString("\\N", ostr);
else
nested_data_type->serializeTextXML(col.getNestedColumn(), row_num, ostr, settings);
nested_data_type->serializeAsTextXML(col.getNestedColumn(), row_num, ostr, settings);
}
void DataTypeNullable::serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf) const

2
dbms/src/DataTypes/DataTypeNullable.h
View File

@ -14,7 +14,7 @@ public:
static constexpr bool is_parametric = true;
explicit DataTypeNullable(const DataTypePtr & nested_data_type_);
std::string getName() const override { return "Nullable(" + nested_data_type->getName() + ")"; }
std::string doGetName() const override { return "Nullable(" + nested_data_type->getName() + ")"; }
const char * getFamilyName() const override { return "Nullable"; }
TypeIndex getTypeId() const override { return TypeIndex::Nullable; }

16
dbms/src/DataTypes/DataTypeTuple.cpp
View File

@ -64,7 +64,7 @@ DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_)
std::string DataTypeTuple::getName() const
std::string DataTypeTuple::doGetName() const
{
size_t size = elems.size();
WriteBufferFromOwnString s;
@ -160,7 +160,7 @@ void DataTypeTuple::serializeText(const IColumn & column, size_t row_num, WriteB
{
if (i != 0)
writeChar(',', ostr);
elems[i]->serializeTextQuoted(extractElementColumn(column, i), row_num, ostr, settings);
elems[i]->serializeAsTextQuoted(extractElementColumn(column, i), row_num, ostr, settings);
}
writeChar(')', ostr);
}
@ -180,7 +180,7 @@ void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr, const F
assertChar(',', istr);
skipWhitespaceIfAny(istr);
}
elems[i]->deserializeTextQuoted(extractElementColumn(column, i), istr, settings);
elems[i]->deserializeAsTextQuoted(extractElementColumn(column, i), istr, settings);
}
});
@ -195,7 +195,7 @@ void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, Wr
{
if (i != 0)
writeChar(',', ostr);
elems[i]->serializeTextJSON(extractElementColumn(column, i), row_num, ostr, settings);
elems[i]->serializeAsTextJSON(extractElementColumn(column, i), row_num, ostr, settings);
}
writeChar(']', ostr);
}
@ -215,7 +215,7 @@ void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, con
assertChar(',', istr);
skipWhitespaceIfAny(istr);
}
elems[i]->deserializeTextJSON(extractElementColumn(column, i), istr, settings);
elems[i]->deserializeAsTextJSON(extractElementColumn(column, i), istr, settings);
}
});
@ -229,7 +229,7 @@ void DataTypeTuple::serializeTextXML(const IColumn & column, size_t row_num, Wri
for (const auto i : ext::range(0, ext::size(elems)))
{
writeCString("<elem>", ostr);
elems[i]->serializeTextXML(extractElementColumn(column, i), row_num, ostr, settings);
elems[i]->serializeAsTextXML(extractElementColumn(column, i), row_num, ostr, settings);
writeCString("</elem>", ostr);
}
writeCString("</tuple>", ostr);
@ -241,7 +241,7 @@ void DataTypeTuple::serializeTextCSV(const IColumn & column, size_t row_num, Wri
{
if (i != 0)
writeChar(',', ostr);
elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr, settings);
elems[i]->serializeAsTextCSV(extractElementColumn(column, i), row_num, ostr, settings);
}
}
@ -258,7 +258,7 @@ void DataTypeTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, cons
assertChar(settings.csv.delimiter, istr);
skipWhitespaceIfAny(istr);
}
elems[i]->deserializeTextCSV(extractElementColumn(column, i), istr, settings);
elems[i]->deserializeAsTextCSV(extractElementColumn(column, i), istr, settings);
}
});
}

2
dbms/src/DataTypes/DataTypeTuple.h
View File

@ -29,7 +29,7 @@ public:
DataTypeTuple(const DataTypes & elems, const Strings & names);
TypeIndex getTypeId() const override { return TypeIndex::Tuple; }
std::string getName() const override;
std::string doGetName() const override;
const char * getFamilyName() const override { return "Tuple"; }
bool canBeInsideNullable() const override { return false; }

2
dbms/src/DataTypes/DataTypesDecimal.cpp
View File

@ -28,7 +28,7 @@ bool decimalCheckArithmeticOverflow(const Context & context) { return context.ge
//
template <typename T>
std::string DataTypeDecimal<T>::getName() const
std::string DataTypeDecimal<T>::doGetName() const
{
std::stringstream ss;
ss << "Decimal(" << precision << ", " << scale << ")";

2
dbms/src/DataTypes/DataTypesDecimal.h
View File

@ -86,7 +86,7 @@ public:
}
const char * getFamilyName() const override { return "Decimal"; }
std::string getName() const override;
std::string doGetName() const override;
TypeIndex getTypeId() const override { return TypeId<T>::value; }
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

155
dbms/src/DataTypes/IDataType.cpp
View File

@ -9,6 +9,7 @@
#include <IO/WriteHelpers.h>
#include <DataTypes/IDataType.h>
#include <DataTypes/IDataTypeDomain.h>
#include <DataTypes/NestedUtils.h>
@ -22,6 +23,31 @@ namespace ErrorCodes
extern const int DATA_TYPE_CANNOT_BE_PROMOTED;
}
IDataType::IDataType()
: domain(nullptr)
{
}
IDataType::~IDataType()
{
}
String IDataType::getName() const
{
if (domain)
{
return domain->getName();
}
else
{
return doGetName();
}
}
String IDataType::doGetName() const
{
return getFamilyName();
}
void IDataType::updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint)
{
@ -114,4 +140,133 @@ void IDataType::insertDefaultInto(IColumn & column) const
column.insertDefault();
}
void IDataType::serializeAsTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeTextEscaped(column, row_num, ostr, settings);
}
else
{
serializeTextEscaped(column, row_num, ostr, settings);
}
}
void IDataType::deserializeAsTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (domain)
{
domain->deserializeTextEscaped(column, istr, settings);
}
else
{
deserializeTextEscaped(column, istr, settings);
}
}
void IDataType::serializeAsTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeTextQuoted(column, row_num, ostr, settings);
}
else
{
serializeTextQuoted(column, row_num, ostr, settings);
}
}
void IDataType::deserializeAsTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (domain)
{
domain->deserializeTextQuoted(column, istr, settings);
}
else
{
deserializeTextQuoted(column, istr, settings);
}
}
void IDataType::serializeAsTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeTextCSV(column, row_num, ostr, settings);
}
else
{
serializeTextCSV(column, row_num, ostr, settings);
}
}
void IDataType::deserializeAsTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (domain)
{
domain->deserializeTextCSV(column, istr, settings);
}
else
{
deserializeTextCSV(column, istr, settings);
}
}
void IDataType::serializeAsText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeText(column, row_num, ostr, settings);
}
else
{
serializeText(column, row_num, ostr, settings);
}
}
void IDataType::serializeAsTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeTextJSON(column, row_num, ostr, settings);
}
else
{
serializeTextJSON(column, row_num, ostr, settings);
}
}
void IDataType::deserializeAsTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (domain)
{
domain->deserializeTextJSON(column, istr, settings);
}
else
{
deserializeTextJSON(column, istr, settings);
}
}
void IDataType::serializeAsTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
if (domain)
{
domain->serializeTextXML(column, row_num, ostr, settings);
}
else
{
serializeTextXML(column, row_num, ostr, settings);
}
}
void IDataType::setDomain(const IDataTypeDomain* const new_domain) const
{
if (domain != nullptr)
{
throw Exception("Type " + getName() + " already has a domain.", ErrorCodes::LOGICAL_ERROR);
}
domain = new_domain;
}
}

68
dbms/src/DataTypes/IDataType.h
View File

@ -12,6 +12,7 @@ namespace DB
class ReadBuffer;
class WriteBuffer;
class IDataTypeDomain;
class IDataType;
struct FormatSettings;
@ -35,6 +36,9 @@ class ProtobufWriter;
class IDataType : private boost::noncopyable
{
public:
IDataType();
virtual ~IDataType();
/// Compile time flag. If false, then if C++ types are the same, then SQL types are also the same.
/// Example: DataTypeString is not parametric: thus all instances of DataTypeString are the same SQL type.
/// Example: DataTypeFixedString is parametric: different instances of DataTypeFixedString may be different SQL types.
@ -42,7 +46,7 @@ public:
/// static constexpr bool is_parametric = false;
/// Name of data type (examples: UInt64, Array(String)).
virtual String getName() const { return getFamilyName(); }
String getName() const;
/// Name of data type family (example: FixedString, Array).
virtual const char * getFamilyName() const = 0;
@ -217,6 +221,43 @@ public:
/// If method will throw an exception, then column will be in same state as before call to method.
virtual void deserializeBinary(IColumn & column, ReadBuffer & istr) const = 0;
/** Text serialization with escaping but without quoting.
*/
virtual void serializeAsTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const;
virtual void deserializeAsTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
/** Text serialization as a literal that may be inserted into a query.
*/
virtual void serializeAsTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const;
virtual void deserializeAsTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
/** Text serialization for the CSV format.
*/
virtual void serializeAsTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const;
virtual void deserializeAsTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
/** Text serialization for displaying on a terminal or saving into a text file, and the like.
* Without escaping or quoting.
*/
virtual void serializeAsText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const;
/** Text serialization intended for using in JSON format.
*/
virtual void serializeAsTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const;
virtual void deserializeAsTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
/** Text serialization for putting into the XML format.
*/
virtual void serializeAsTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const;
/** Serialize to a protobuf. */
virtual void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf) const = 0;
protected:
virtual String doGetName() const;
/** Text serialization with escaping but without quoting.
*/
virtual void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
@ -232,10 +273,6 @@ public:
/** Text serialization for the CSV format.
*/
virtual void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
/** delimiter - the delimiter we expect when reading a string value that is not double-quoted
* (the delimiter is not consumed).
*/
virtual void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization for displaying on a terminal or saving into a text file, and the like.
@ -256,9 +293,7 @@ public:
serializeText(column, row_num, ostr, settings);
}
/** Serialize to a protobuf. */
virtual void serializeProtobuf(const IColumn & column, size_t row_num, ProtobufWriter & protobuf) const = 0;
public:
/** Create empty column for corresponding type.
*/
virtual MutableColumnPtr createColumn() const = 0;
@ -290,8 +325,6 @@ public:
/// Checks that two instances belong to the same type
virtual bool equals(const IDataType & rhs) const = 0;
virtual ~IDataType() {}
/// Various properties on behaviour of data type.
@ -419,6 +452,21 @@ public:
static void updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint);
static String getFileNameForStream(const String & column_name, const SubstreamPath & path);
private:
friend class DataTypeFactory;
/** Sets domain on existing DataType, can be considered as second phase
* of construction explicitly done by DataTypeFactory.
* Will throw an exception if domain is already set.
*/
void setDomain(const IDataTypeDomain* newDomain) const;
private:
/** This is mutable to allow setting domain on `const IDataType` post construction,
* simplifying creation of domains for all types, without them even knowing
* of domain existence.
*/
mutable IDataTypeDomain const* domain;
};

59
dbms/src/DataTypes/IDataTypeDomain.h Normal file
View File

@ -0,0 +1,59 @@
#pragma once
#include <cstddef>
namespace DB
{
class ReadBuffer;
class WriteBuffer;
struct FormatSettings;
class IColumn;
/** Further refinment of the properties of data type.
*
* Contains methods for serialization/deserialization.
* Implementations of this interface represent a data type domain (example: IPv4)
* which is a refinement of the exsitgin type with a name and specific text
* representation.
*
* IDataTypeDomain is totally immutable object. You can always share them.
*/
class IDataTypeDomain
{
public:
virtual ~IDataTypeDomain() {}
virtual const char* getName() const = 0;
/** Text serialization for displaying on a terminal or saving into a text file, and the like.
* Without escaping or quoting.
*/
virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
/** Text serialization with escaping but without quoting.
*/
virtual void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
virtual void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization as a literal that may be inserted into a query.
*/
virtual void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
virtual void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization for the CSV format.
*/
virtual void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
virtual void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization intended for using in JSON format.
*/
virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/** Text serialization for putting into the XML format.
*/
virtual void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const = 0;
};
} // namespace DB

2
dbms/src/Dictionaries/DictionaryStructure.cpp
View File

@ -355,7 +355,7 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
{
ReadBufferFromString null_value_buffer{null_value_string};
auto column_with_null_value = type->createColumn();
type->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
type->deserializeAsTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
null_value = (*column_with_null_value)[0];
}
}

4
dbms/src/Dictionaries/ExternalQueryBuilder.cpp
View File

@ -345,7 +345,7 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons
/// key_i=value_i
writeString(key_description.name, out);
writeString("=", out);
key_description.type->serializeTextQuoted(*key_columns[i], row, out, format_settings);
key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
}
writeString(")", out);
@ -387,7 +387,7 @@ void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const si
writeString(", ", out);
first = false;
(*dict_struct.key)[i].type->serializeTextQuoted(*key_columns[i], row, out, format_settings);
(*dict_struct.key)[i].type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
}
writeString(")", out);

4
dbms/src/Formats/CSVRowInputStream.cpp
View File

@ -123,7 +123,7 @@ bool CSVRowInputStream::read(MutableColumns & columns, RowReadExtension &)
for (size_t i = 0; i < size; ++i)
{
skipWhitespacesAndTabs(istr);
data_types[i]->deserializeTextCSV(*columns[i], istr, format_settings);
data_types[i]->deserializeAsTextCSV(*columns[i], istr, format_settings);
skipWhitespacesAndTabs(istr);
skipDelimiter(istr, format_settings.csv.delimiter, i + 1 == size);
@ -215,7 +215,7 @@ bool CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumns & columns,
{
skipWhitespacesAndTabs(istr);
prev_position = istr.position();
data_types[i]->deserializeTextCSV(*columns[i], istr, format_settings);
data_types[i]->deserializeAsTextCSV(*columns[i], istr, format_settings);
curr_position = istr.position();
skipWhitespacesAndTabs(istr);
}

2
dbms/src/Formats/CSVRowOutputStream.cpp
View File

@ -42,7 +42,7 @@ void CSVRowOutputStream::writePrefix()
void CSVRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextCSV(column, row_num, ostr, format_settings);
type.serializeAsTextCSV(column, row_num, ostr, format_settings);
}

6
dbms/src/Formats/JSONCompactRowOutputStream.cpp
View File

@ -16,7 +16,7 @@ JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, cons
void JSONCompactRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextJSON(column, row_num, *ostr, settings);
type.serializeAsTextJSON(column, row_num, *ostr, settings);
++field_number;
}
@ -58,7 +58,7 @@ void JSONCompactRowOutputStream::writeTotals()
writeChar(',', *ostr);
const ColumnWithTypeAndName & column = totals.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
column.type->serializeAsTextJSON(*column.column.get(), 0, *ostr, settings);
}
writeChar(']', *ostr);
@ -79,7 +79,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeChar(',', ostr);
const ColumnWithTypeAndName & column = extremes.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
column.type->serializeAsTextJSON(*column.column.get(), row_num, ostr, settings);
}
writeChar(']', ostr);

2
dbms/src/Formats/JSONEachRowRowInputStream.cpp
View File

@ -134,7 +134,7 @@ void JSONEachRowRowInputStream::readField(size_t index, MutableColumns & columns
try
{
header.getByPosition(index).type->deserializeTextJSON(*columns[index], istr, format_settings);
header.getByPosition(index).type->deserializeAsTextJSON(*columns[index], istr, format_settings);
}
catch (Exception & e)
{

2
dbms/src/Formats/JSONEachRowRowOutputStream.cpp
View File

@ -27,7 +27,7 @@ void JSONEachRowRowOutputStream::writeField(const IColumn & column, const IDataT
{
writeString(fields[field_number], ostr);
writeChar(':', ostr);
type.serializeTextJSON(column, row_num, ostr, settings);
type.serializeAsTextJSON(column, row_num, ostr, settings);
++field_number;
}

6
dbms/src/Formats/JSONRowOutputStream.cpp
View File

@ -71,7 +71,7 @@ void JSONRowOutputStream::writeField(const IColumn & column, const IDataType & t
writeCString("\t\t\t", *ostr);
writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr);
type.serializeTextJSON(column, row_num, *ostr, settings);
type.serializeAsTextJSON(column, row_num, *ostr, settings);
++field_number;
}
@ -151,7 +151,7 @@ void JSONRowOutputStream::writeTotals()
writeCString("\t\t", *ostr);
writeJSONString(column.name, *ostr, settings);
writeCString(": ", *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
column.type->serializeAsTextJSON(*column.column.get(), 0, *ostr, settings);
}
writeChar('\n', *ostr);
@ -178,7 +178,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeCString("\t\t\t", ostr);
writeJSONString(column.name, ostr, settings);
writeCString(": ", ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
column.type->serializeAsTextJSON(*column.column.get(), row_num, ostr, settings);
}
writeChar('\n', ostr);

2
dbms/src/Formats/ODBCDriver2BlockOutputStream.cpp
View File

@ -43,7 +43,7 @@ static void writeRow(const Block & block, size_t row_idx, WriteBuffer & out, con
{
{
WriteBufferFromString text_out(buffer);
col.type->serializeText(*col.column, row_idx, text_out, format_settings);
col.type->serializeAsText(*col.column, row_idx, text_out, format_settings);
}
writeODBCString(out, buffer);
}

2
dbms/src/Formats/ODBCDriverBlockOutputStream.cpp
View File

@ -33,7 +33,7 @@ void ODBCDriverBlockOutputStream::write(const Block & block)
{
WriteBufferFromString text_out(text_value);
col.type->serializeText(*col.column, i, text_out, format_settings);
col.type->serializeAsText(*col.column, i, text_out, format_settings);
}
writeStringBinary(text_value, out);

6
dbms/src/Formats/PrettyBlockOutputStream.cpp
View File

@ -58,7 +58,7 @@ void PrettyBlockOutputStream::calculateWidths(
{
{
WriteBufferFromString out(serialized_value);
elem.type->serializeText(*elem.column, j, out, format_settings);
elem.type->serializeAsText(*elem.column, j, out, format_settings);
}
widths[i][j] = std::min<UInt64>(format_settings.pretty.max_column_pad_width,
@ -206,11 +206,11 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName
if (elem.type->shouldAlignRightInPrettyFormats())
{
writePadding();
elem.type->serializeText(*elem.column.get(), row_num, ostr, format_settings);
elem.type->serializeAsText(*elem.column.get(), row_num, ostr, format_settings);
}
else
{
elem.type->serializeText(*elem.column.get(), row_num, ostr, format_settings);
elem.type->serializeAsText(*elem.column.get(), row_num, ostr, format_settings);
writePadding();
}
}

2
dbms/src/Formats/TSKVRowInputStream.cpp
View File

@ -135,7 +135,7 @@ bool TSKVRowInputStream::read(MutableColumns & columns, RowReadExtension & ext)
read_columns[index] = true;
header.getByPosition(index).type->deserializeTextEscaped(*columns[index], istr, format_settings);
header.getByPosition(index).type->deserializeAsTextEscaped(*columns[index], istr, format_settings);
}
}
else

2
dbms/src/Formats/TSKVRowOutputStream.cpp
View File

@ -28,7 +28,7 @@ TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & samp
void TSKVRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
writeString(fields[field_number].name, ostr);
type.serializeTextEscaped(column, row_num, ostr, format_settings);
type.serializeAsTextEscaped(column, row_num, ostr, format_settings);
++field_number;
}

2
dbms/src/Formats/TabSeparatedRawRowOutputStream.h
View File

@ -18,7 +18,7 @@ public:
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override
{
type.serializeText(column, row_num, ostr, format_settings);
type.serializeAsText(column, row_num, ostr, format_settings);
}
};

4
dbms/src/Formats/TabSeparatedRowInputStream.cpp
View File

@ -86,7 +86,7 @@ bool TabSeparatedRowInputStream::read(MutableColumns & columns, RowReadExtension
for (size_t i = 0; i < size; ++i)
{
data_types[i]->deserializeTextEscaped(*columns[i], istr, format_settings);
data_types[i]->deserializeAsTextEscaped(*columns[i], istr, format_settings);
/// skip separators
if (i + 1 == size)
@ -183,7 +183,7 @@ bool TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumns &
try
{
data_types[i]->deserializeTextEscaped(*columns[i], istr, format_settings);
data_types[i]->deserializeAsTextEscaped(*columns[i], istr, format_settings);
}
catch (...)
{

2
dbms/src/Formats/TabSeparatedRowOutputStream.cpp
View File

@ -48,7 +48,7 @@ void TabSeparatedRowOutputStream::writePrefix()
void TabSeparatedRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextEscaped(column, row_num, ostr, format_settings);
type.serializeAsTextEscaped(column, row_num, ostr, format_settings);
}

2
dbms/src/Formats/ValuesRowInputStream.cpp
View File

@ -64,7 +64,7 @@ bool ValuesRowInputStream::read(MutableColumns & columns, RowReadExtension &)
bool rollback_on_exception = false;
try
{
header.getByPosition(i).type->deserializeTextQuoted(*columns[i], istr, format_settings);
header.getByPosition(i).type->deserializeAsTextQuoted(*columns[i], istr, format_settings);
rollback_on_exception = true;
skipWhitespaceIfAny(istr);

2
dbms/src/Formats/ValuesRowOutputStream.cpp
View File

@ -23,7 +23,7 @@ void ValuesRowOutputStream::flush()
void ValuesRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextQuoted(column, row_num, ostr, format_settings);
type.serializeAsTextQuoted(column, row_num, ostr, format_settings);
}
void ValuesRowOutputStream::writeFieldDelimiter()

2
dbms/src/Formats/VerticalRowOutputStream.cpp
View File

@ -71,7 +71,7 @@ void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType
void VerticalRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
{
type.serializeText(column, row_num, ostr, format_settings);
type.serializeAsText(column, row_num, ostr, format_settings);
}

6
dbms/src/Formats/XMLRowOutputStream.cpp
View File

@ -87,7 +87,7 @@ void XMLRowOutputStream::writeField(const IColumn & column, const IDataType & ty
writeCString("\t\t\t<", *ostr);
writeString(field_tag_names[field_number], *ostr);
writeCString(">", *ostr);
type.serializeTextXML(column, row_num, *ostr, format_settings);
type.serializeAsTextXML(column, row_num, *ostr, format_settings);
writeCString("</", *ostr);
writeString(field_tag_names[field_number], *ostr);
writeCString(">\n", *ostr);
@ -153,7 +153,7 @@ void XMLRowOutputStream::writeTotals()
writeCString("\t\t<", *ostr);
writeString(field_tag_names[i], *ostr);
writeCString(">", *ostr);
column.type->serializeTextXML(*column.column.get(), 0, *ostr, format_settings);
column.type->serializeAsTextXML(*column.column.get(), 0, *ostr, format_settings);
writeCString("</", *ostr);
writeString(field_tag_names[i], *ostr);
writeCString(">\n", *ostr);
@ -179,7 +179,7 @@ static void writeExtremesElement(
writeCString("\t\t\t<", ostr);
writeString(field_tag_names[i], ostr);
writeCString(">", ostr);
column.type->serializeTextXML(*column.column.get(), row_num, ostr, format_settings);
column.type->serializeAsTextXML(*column.column.get(), row_num, ostr, format_settings);
writeCString("</", ostr);
writeString(field_tag_names[i], ostr);
writeCString(">\n", ostr);

2
dbms/src/Functions/FunctionsCoding.cpp
View File

@ -26,6 +26,8 @@ void registerFunctionsCoding(FunctionFactory & factory)
factory.registerFunction<FunctionHex>();
factory.registerFunction<FunctionUnhex>();
factory.registerFunction<FunctionBitmaskToArray>();
factory.registerFunction<FunctionToIPv4>();
factory.registerFunction<FunctionToIPv6>();
}
}

303
dbms/src/Functions/FunctionsCoding.h
View File

@ -4,6 +4,7 @@
#include <Common/formatIPv6.h>
#include <Common/typeid_cast.h>
#include <IO/WriteHelpers.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeFixedString.h>
@ -54,8 +55,6 @@ namespace ErrorCodes
*/
constexpr size_t ipv4_bytes_length = 4;
constexpr size_t ipv6_bytes_length = 16;
constexpr size_t uuid_bytes_length = 16;
constexpr size_t uuid_text_length = 36;
@ -74,10 +73,10 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != ipv6_bytes_length)
if (!ptr || ptr->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument of function " + getName() +
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeString>();
@ -92,11 +91,11 @@ public:
if (const auto col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in->getN() != ipv6_bytes_length)
if (col_in->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + col_type_name.type->getName() +
" of column " + col_in->getName() +
" argument of function " + getName() +
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto size = col_in->size();
@ -112,7 +111,7 @@ public:
auto begin = reinterpret_cast<char *>(vec_res.data());
auto pos = begin;
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += IPV6_BINARY_LENGTH, ++i)
{
formatIPv6(&vec_in[offset], pos);
offsets_res[i] = pos - begin;
@ -143,10 +142,10 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto ptr = checkAndGetDataType<DataTypeFixedString>(arguments[0].get());
if (!ptr || ptr->getN() != ipv6_bytes_length)
if (!ptr || ptr->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + arguments[0]->getName() +
" of argument 1 of function " + getName() +
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (!WhichDataType(arguments[1]).isUInt8())
@ -177,11 +176,11 @@ public:
if (const auto col_in = checkAndGetColumn<ColumnFixedString>(column.get()))
{
if (col_in->getN() != ipv6_bytes_length)
if (col_in->getN() != IPV6_BINARY_LENGTH)
throw Exception("Illegal type " + col_type_name.type->getName() +
" of column " + col_in->getName() +
" argument of function " + getName() +
", expected FixedString(" + toString(ipv6_bytes_length) + ")",
", expected FixedString(" + toString(IPV6_BINARY_LENGTH) + ")",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto ipv6_zeroed_tail_bytes = checkAndGetColumnConst<ColumnVector<UInt8>>(col_ipv6_zeroed_tail_bytes.get());
@ -191,7 +190,7 @@ public:
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
UInt8 ipv6_zeroed_tail_bytes_count = ipv6_zeroed_tail_bytes->getValue<UInt8>();
if (ipv6_zeroed_tail_bytes_count > ipv6_bytes_length)
if (ipv6_zeroed_tail_bytes_count > IPV6_BINARY_LENGTH)
throw Exception("Illegal value for argument 2 " + col_ipv6_zeroed_tail_bytes_type.type->getName() +
" of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -203,7 +202,7 @@ public:
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
UInt8 ipv4_zeroed_tail_bytes_count = ipv4_zeroed_tail_bytes->getValue<UInt8>();
if (ipv4_zeroed_tail_bytes_count > ipv6_bytes_length)
if (ipv4_zeroed_tail_bytes_count > IPV6_BINARY_LENGTH)
throw Exception("Illegal value for argument 3 " + col_ipv4_zeroed_tail_bytes_type.type->getName() +
" of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -221,7 +220,7 @@ public:
auto begin = reinterpret_cast<char *>(vec_res.data());
auto pos = begin;
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += ipv6_bytes_length, ++i)
for (size_t offset = 0, i = 0; offset < vec_in.size(); offset += IPV6_BINARY_LENGTH, ++i)
{
const auto address = &vec_in[offset];
UInt8 zeroed_tail_bytes_count = isIPv4Mapped(address) ? ipv4_zeroed_tail_bytes_count : ipv6_zeroed_tail_bytes_count;
@ -269,146 +268,7 @@ public:
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeFixedString>(ipv6_bytes_length);
}
static bool ipv4_scan(const char * src, unsigned char * dst)
{
constexpr auto size = sizeof(UInt32);
char bytes[size]{};
for (const auto i : ext::range(0, size))
{
UInt32 value = 0;
size_t len = 0;
while (isNumericASCII(*src) && len <= 3)
{
value = value * 10 + (*src - '0');
++len;
++src;
}
if (len == 0 || value > 255 || (i < size - 1 && *src != '.'))
{
memset(dst, 0, size);
return false;
}
bytes[i] = value;
++src;
}
if (src[-1] != '\0')
{
memset(dst, 0, size);
return false;
}
memcpy(dst, bytes, sizeof(bytes));
return true;
}
/// slightly altered implementation from http://svn.apache.org/repos/asf/apr/apr/trunk/network_io/unix/inet_pton.c
static void ipv6_scan(const char * src, unsigned char * dst)
{
const auto clear_dst = [dst]
{
memset(dst, '\0', ipv6_bytes_length);
};
/// Leading :: requires some special handling.
if (*src == ':')
if (*++src != ':')
return clear_dst();
unsigned char tmp[ipv6_bytes_length]{};
auto tp = tmp;
auto endp = tp + ipv6_bytes_length;
auto curtok = src;
auto saw_xdigit = false;
UInt32 val{};
unsigned char * colonp = nullptr;
/// Assuming zero-terminated string.
while (const auto ch = *src++)
{
const auto num = unhex(ch);
if (num != '\xff')
{
val <<= 4;
val |= num;
if (val > 0xffffu)
return clear_dst();
saw_xdigit = 1;
continue;
}
if (ch == ':')
{
curtok = src;
if (!saw_xdigit)
{
if (colonp)
return clear_dst();
colonp = tp;
continue;
}
if (tp + sizeof(UInt16) > endp)
return clear_dst();
*tp++ = static_cast<unsigned char>((val >> 8) & 0xffu);
*tp++ = static_cast<unsigned char>(val & 0xffu);
saw_xdigit = false;
val = 0;
continue;
}
if (ch == '.' && (tp + ipv4_bytes_length) <= endp)
{
if (!ipv4_scan(curtok, tp))
return clear_dst();
tp += ipv4_bytes_length;
saw_xdigit = false;
break; /* '\0' was seen by ipv4_scan(). */
}
return clear_dst();
}
if (saw_xdigit)
{
if (tp + sizeof(UInt16) > endp)
return clear_dst();
*tp++ = static_cast<unsigned char>((val >> 8) & 0xffu);
*tp++ = static_cast<unsigned char>(val & 0xffu);
}
if (colonp)
{
/*
* Since some memmove()'s erroneously fail to handle
* overlapping regions, we'll do the shift by hand.
*/
const auto n = tp - colonp;
for (int i = 1; i <= n; ++i)
{
endp[- i] = colonp[n - i];
colonp[n - i] = 0;
}
tp = endp;
}
if (tp != endp)
return clear_dst();
memcpy(dst, tmp, sizeof(tmp));
return std::make_shared<DataTypeFixedString>(IPV6_BINARY_LENGTH);
}
bool useDefaultImplementationForConstants() const override { return true; }
@ -419,10 +279,10 @@ public:
if (const auto col_in = checkAndGetColumn<ColumnString>(column.get()))
{
auto col_res = ColumnFixedString::create(ipv6_bytes_length);
auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH);
auto & vec_res = col_res->getChars();
vec_res.resize(col_in->size() * ipv6_bytes_length);
vec_res.resize(col_in->size() * IPV6_BINARY_LENGTH);
const ColumnString::Chars & vec_src = col_in->getChars();
const ColumnString::Offsets & offsets_src = col_in->getOffsets();
@ -430,9 +290,10 @@ public:
for (size_t out_offset = 0, i = 0;
out_offset < vec_res.size();
out_offset += ipv6_bytes_length, ++i)
out_offset += IPV6_BINARY_LENGTH, ++i)
{
ipv6_scan(reinterpret_cast<const char * >(&vec_src[src_offset]), &vec_res[out_offset]);
//TODO(nemkov): handle failure ?
parseIPv6(reinterpret_cast<const char *>(&vec_src[src_offset]), &vec_res[out_offset]);
src_offset = offsets_src[i];
}
@ -451,59 +312,6 @@ public:
template <size_t mask_tail_octets, typename Name>
class FunctionIPv4NumToString : public IFunction
{
private:
static void formatIP(UInt32 ip, char *& out)
{
char * begin = out;
for (size_t octet = 0; octet < mask_tail_octets; ++octet)
{
if (octet > 0)
{
*out = '.';
++out;
}
memcpy(out, "xxx", 3); /// Strange choice, but meets the specification.
out += 3;
}
/// Write everything backwards. NOTE The loop is unrolled.
for (size_t octet = mask_tail_octets; octet < 4; ++octet)
{
if (octet > 0)
{
*out = '.';
++out;
}
/// Get the next byte.
UInt32 value = (ip >> (octet * 8)) & static_cast<UInt32>(0xFF);
/// Faster than sprintf. NOTE Actually not good enough. LUT will be better.
if (value == 0)
{
*out = '0';
++out;
}
else
{
while (value > 0)
{
*out = '0' + value % 10;
++out;
value /= 10;
}
}
}
/// And reverse.
std::reverse(begin, out);
*out = '\0';
++out;
}
public:
static constexpr auto name = Name::name;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionIPv4NumToString<mask_tail_octets, Name>>(); }
@ -547,7 +355,7 @@ public:
for (size_t i = 0; i < vec_in.size(); ++i)
{
formatIP(vec_in[i], pos);
DB::formatIPv4(reinterpret_cast<const unsigned char*>(&vec_in[i]), pos, mask_tail_octets, "xxx");
offsets_res[i] = pos - begin;
}
@ -585,27 +393,12 @@ public:
return std::make_shared<DataTypeUInt32>();
}
static UInt32 parseIPv4(const char * pos)
static inline UInt32 parseIPv4(const char * pos)
{
UInt32 res = 0;
for (int offset = 24; offset >= 0; offset -= 8)
{
UInt32 value = 0;
size_t len = 0;
while (isNumericASCII(*pos) && len <= 3)
{
value = value * 10 + (*pos - '0');
++len;
++pos;
}
if (len == 0 || value > 255 || (offset > 0 && *pos != '.'))
return 0;
res |= value << offset;
++pos;
}
if (*(pos - 1) != '\0')
return 0;
return res;
UInt32 result = 0;
DB::parseIPv4(pos, reinterpret_cast<unsigned char*>(&result));
return result;
}
bool useDefaultImplementationForConstants() const override { return true; }
@ -670,14 +463,14 @@ public:
if (const auto col_in = typeid_cast<const ColumnUInt32 *>(column.get()))
{
auto col_res = ColumnFixedString::create(ipv6_bytes_length);
auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH);
auto & vec_res = col_res->getChars();
vec_res.resize(col_in->size() * ipv6_bytes_length);
vec_res.resize(col_in->size() * IPV6_BINARY_LENGTH);
const auto & vec_in = col_in->getData();
for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += ipv6_bytes_length, ++i)
for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += IPV6_BINARY_LENGTH, ++i)
mapIPv4ToIPv6(vec_in[i], &vec_res[out_offset]);
block.getByPosition(result).column = std::move(col_res);
@ -696,6 +489,46 @@ private:
}
};
class FunctionToIPv4 : public FunctionIPv4StringToNum
{
public:
static constexpr auto name = "toIPv4";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionToIPv4>(); }
String getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!isString(arguments[0]))
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return DataTypeFactory::instance().get("IPv4");
}
};
class FunctionToIPv6 : public FunctionIPv6StringToNum
{
public:
static constexpr auto name = "toIPv6";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionToIPv6>(); }
String getName() const override { return name; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!isString(arguments[0]))
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return DataTypeFactory::instance().get("IPv6");
}
};
class FunctionMACNumToString : public IFunction
{

4
dbms/src/Functions/FunctionsConversion.h
View File

@ -347,7 +347,7 @@ struct ConvertImplGenericToString
FormatSettings format_settings;
for (size_t i = 0; i < size; ++i)
{
type.serializeText(col_from, i, write_buffer, format_settings);
type.serializeAsText(col_from, i, write_buffer, format_settings);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
@ -631,7 +631,7 @@ struct ConvertImplGenericFromString
{
ReadBufferFromMemory read_buffer(&chars[current_offset], offsets[i] - current_offset - 1);
data_type_to.deserializeTextEscaped(column_to, read_buffer, format_settings);
data_type_to.deserializeAsTextEscaped(column_to, read_buffer, format_settings);
if (!read_buffer.eof())
throwExceptionForIncompletelyParsedValue(read_buffer, block, result);

2
dbms/src/Functions/visibleWidth.cpp
View File

@ -60,7 +60,7 @@ public:
{
{
WriteBufferFromString out(tmp);
src.type->serializeText(*src.column, i, out, format_settings);
src.type->serializeAsText(*src.column, i, out, format_settings);
}
res_data[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(tmp.data()), tmp.size());

9
dbms/src/Interpreters/convertFieldToType.cpp
View File

@ -249,6 +249,15 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
}
}
if (src.getType() == Field::Types::String)
{
const auto col = type.createColumn();
ReadBufferFromString buffer(src.get<String>());
type.deserializeAsTextEscaped(*col, buffer, FormatSettings{});
return (*col)[0];
}
// TODO (nemkov): should we attempt to parse value using or `type.deserializeAsTextEscaped()` type.deserializeAsTextEscaped() ?
throw Exception("Type mismatch in IN or VALUES section. Expected: " + type.getName() + ". Got: "
+ Field::Types::toString(src.getType()), ErrorCodes::TYPE_MISMATCH);
}

2
dbms/src/Storages/MergeTree/MergeTreePartition.cpp
View File

@ -98,7 +98,7 @@ void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffe
const DataTypePtr & type = storage.partition_key_sample.getByPosition(0).type;
auto column = type->createColumn();
column->insert(value[0]);
type->serializeText(*column, 0, out, format_settings);
type->serializeAsText(*column, 0, out, format_settings);
}
else
{

152
dbms/tests/performance/ipv4_ipv6/IPv4.xml Normal file
View File

@ -0,0 +1,152 @@
<test>
<name>IPv4 Functions</name>
<type>once</type>
<tags>
</tags>
<stop_conditions>
<any_of>
<average_speed_not_changing_for_ms>2000</average_speed_not_changing_for_ms>
<total_time_ms>10000</total_time_ms>
</any_of>
</stop_conditions>
<metrics>
<max_rows_per_second />
<max_bytes_per_second />
<avg_rows_per_second />
<avg_bytes_per_second />
</metrics>
<substitutions>
<substitution>
<name>ipv4_string</name>
<values>
<!-- The CAIDA UCSD IPv4 Routed /24 DNS Names Dataset – 20171130,
http://www.caida.org/data/active/ipv4_dnsnames_dataset.xml.
Randomly selected entries from first 50000 rows of dataset. -->
<value>116.253.40.133</value>
<value>183.247.232.58</value>
<value>116.106.34.242</value>
<value>111.56.27.171</value>
<value>183.245.137.140</value>
<value>183.212.25.70</value>
<value>162.144.2.57</value>
<value>111.4.229.190</value>
<value>59.52.3.168</value>
<value>115.11.21.200</value>
<value>121.28.97.113</value>
<value>111.46.39.248</value>
<value>120.192.122.34</value>
<value>113.56.44.105</value>
<value>116.66.238.92</value>
<value>67.22.254.206</value>
<value>115.0.24.191</value>
<value>182.30.107.86</value>
<value>223.73.153.243</value>
<value>115.159.103.38</value>
<value>36.186.75.121</value>
<value>111.56.188.125</value>
<value>115.14.93.25</value>
<value>211.97.110.141</value>
<value>61.58.96.173</value>
<value>203.126.212.37</value>
<value>192.220.125.142</value>
<value>115.22.20.223</value>
<value>121.25.160.80</value>
<value>117.150.98.199</value>
<value>183.211.172.143</value>
<value>180.244.18.143</value>
<value>209.131.3.252</value>
<value>220.200.1.22</value>
<value>171.225.130.45</value>
<value>115.4.78.200</value>
<value>36.183.59.29</value>
<value>218.42.159.17</value>
<value>115.13.39.164</value>
<value>142.254.161.133</value>
<value>116.2.211.43</value>
<value>36.183.126.25</value>
<value>66.150.171.196</value>
<value>104.149.148.137</value>
<value>120.239.82.212</value>
<value>111.14.182.156</value>
<value>115.6.63.224</value>
<value>153.35.83.233</value>
<value>113.142.1.1</value>
<value>121.25.82.29</value>
<value>62.151.203.189</value>
<value>104.27.46.146</value>
<value>36.189.46.88</value>
<value>116.252.54.207</value>
<value>64.77.240.1</value>
<value>142.252.102.78</value>
<value>36.82.224.170</value>
<value>117.33.191.217</value>
<value>144.12.164.251</value>
<value>122.10.93.66</value>
<value>104.25.84.59</value>
<value>111.4.242.106</value>
<value>222.216.51.186</value>
<value>112.33.13.212</value>
<value>115.9.240.116</value>
<value>171.228.0.153</value>
<value>45.3.47.158</value>
<value>69.57.193.230</value>
<value>115.6.104.199</value>
<value>104.24.237.140</value>
<value>199.17.84.108</value>
<value>120.193.17.57</value>
<value>112.40.38.145</value>
<value>67.55.90.43</value>
<value>180.253.57.249</value>
<value>14.204.253.158</value>
<value>1.83.241.116</value>
<value>202.198.37.147</value>
<value>115.6.31.95</value>
<value>117.32.14.179</value>
<value>23.238.237.26</value>
<value>116.97.76.104</value>
<value>1.80.2.248</value>
<value>59.50.185.152</value>
<value>42.117.228.166</value>
<value>119.36.22.147</value>
<value>210.66.18.184</value>
<value>115.19.192.159</value>
<value>112.15.128.113</value>
<value>1.55.138.211</value>
<value>210.183.19.113</value>
<value>42.115.43.114</value>
<value>58.16.171.31</value>
<value>171.234.78.185</value>
<value>113.56.43.134</value>
<value>111.53.182.225</value>
<value>107.160.215.141</value>
<value>171.229.231.90</value>
<value>58.19.84.138</value>
<value>36.79.88.107</value>
<!-- invalid values -->
<value tag="error"></value>
<value tag="error"> </value>
<value tag="error">1</value>
<value tag="error">1.</value>
<value tag="error">1.2.</value>
<value tag="error">.2.</value>
<value tag="error">abc</value>
<value tag="error">127.0.0.1/24</value>
<value tag="error"> 127.0.0.1</value>
<value tag="error">127.0.0.1 </value>
<value tag="error">127.0.0.1?</value>
<value tag="error">999.999.999.999</value>
</values>
</substitution>
</substitutions>
<query tag='IPv4StringToNum'>SELECT count() FROM system.numbers WHERE NOT ignore(IPv4StringToNum(materialize('{ipv4_string}')))</query>
<query tag='IPv4NumToString+IPv4StringToNum'>SELECT count() FROM system.numbers WHERE NOT ignore(IPv4NumToString(IPv4StringToNum(materialize('{ipv4_string}'))))</query>
<query tag='IPv4NumToStringClassC+IPv4StringToNum'>SELECT count() FROM system.numbers WHERE NOT ignore(IPv4NumToStringClassC(IPv4StringToNum(materialize('{ipv4_string}'))))</query>
<query tag='IPv4ToIPv6+IPv4StringToNum'>SELECT count() FROM system.numbers WHERE NOT ignore(IPv4ToIPv6(IPv4StringToNum(materialize('{ipv4_string}'))))</query>
</test>

257
dbms/tests/performance/ipv4_ipv6/IPv6.xml Normal file
View File

@ -0,0 +1,257 @@
<test>
<name>IPv6 Functions</name>
<type>once</type>
<tags>
</tags>
<stop_conditions>
<any_of>
<average_speed_not_changing_for_ms>2000</average_speed_not_changing_for_ms>
<total_time_ms>10000</total_time_ms>
</any_of>
</stop_conditions>
<metrics>
<max_rows_per_second />
<max_bytes_per_second />
<avg_rows_per_second />
<avg_bytes_per_second />
</metrics>
<substitutions>
<substitution>
<name>ipv6_string</name>
<values>
<!-- The CAIDA UCSD IPv4 Routed /24 DNS Names Dataset - 20181130,
http://www.caida.org/data/active/ipv4_dnsnames_dataset.xml.
Randomly selected entries from first 50000 rows of dataset. -->
<value>2606:2b00::1</value>
<value>2001:2000:3080:1351::2</value>
<value>2a01:8840:16::1</value>
<value>2001:550:0:1000::9a36:2a61</value>
<value>2001:578:400:4:2000::19</value>
<value>2607:f290::1</value>
<value>2a02:23f0:ffff:8::5</value>
<value>2400:c700:0:158::</value>
<value>2001:67c:24e4:4::250</value>
<value>2a02:2a38:37:5::2</value>
<value>2001:41a8:400:2::13a</value>
<value>2405:9800:9800:66::2</value>
<value>2a07:a343:f210::1</value>
<value>2403:5000:171:46::2</value>
<value>2800:c20:1141::8</value>
<value>2402:7800:40:2::62</value>
<value>2a00:de00::1</value>
<value>2001:688:0:2:1::9e</value>
<value>2001:2000:3080:80::2</value>
<value>2001:428::205:171:200:230</value>
<value>2001:fb1:fe0:9::8</value>
<value>2001:2000:3080:10ca::2</value>
<value>2400:dd0b:1003::2</value>
<value>2001:1a98:6677::9d9d:140a</value>
<value>2001:2000:3018:3b::1</value>
<value>2607:fa70:3:33::2</value>
<value>2001:5b0:23ff:fffa::113</value>
<value>2001:450:2001:1000:0:40:6924:23</value>
<value>2001:418:0:5000::c2d</value>
<value>2a01:b740:a09::1</value>
<value>2607:f0d0:2:2::243</value>
<value>2a01:348::e:1:1</value>
<value>2405:4800::3221:3621:2</value>
<value>2a02:aa08:e000:3100::2</value>
<value>2001:44c8:129:2632:33:0:252:2</value>
<value>2a02:e980:1e::1</value>
<value>2a0a:6f40:2::1</value>
<value>2001:550:2:29::2c9:1</value>
<value>2001:c20:4800::175</value>
<value>2c0f:feb0:1:2::d1</value>
<value>2a0b:7086:fff0::1</value>
<value>2a04:2dc0::16d</value>
<value>2604:7e00::105d</value>
<value>2001:470:1:946::2</value>
<value>2a0c:3240::1</value>
<value>2800:630:4010:8::2</value>
<value>2001:1af8:4040::12</value>
<value>2c0f:fc98:1200::2</value>
<value>2001:470:1:248::2</value>
<value>2620:44:a000::1</value>
<value>2402:800:63ff:40::1</value>
<value>2a02:b000:fff::524</value>
<value>2001:470:0:327::1</value>
<value>2401:7400:8888:2::8</value>
<value>2001:500:55::1</value>
<value>2001:668:0:3::f000:c2</value>
<value>2400:bf40::1</value>
<value>2001:67c:754::1</value>
<value>2402:28c0:100:ffff:ffff:ffff:ffff:ffff</value>
<value>2001:470:0:1fa::2</value>
<value>2001:550:0:1000::9a18:292a</value>
<value>2001:470:1:89e::2</value>
<value>2001:579:6f05:500:9934:5b3e:b7fe:1447</value>
<value>2804:158c::1</value>
<value>2600:140e:6::1</value>
<value>2a00:18e0:0:bb04::82</value>
<value>2a02:2698:5000::1e06</value>
<value>2402:800:63ff:10::7:2</value>
<value>2a02:e980:19::1</value>
<value>2001:4888::342:1:0:0</value>
<value>2607:fc68:0:4:0:2:2711:21</value>
<value>2606:2800:602a::1</value>
<value>2404:c600:1000:2::1d1</value>
<value>2001:578:1400:4::9d</value>
<value>2804:64:0:25::1</value>
<value>2605:3e00::1:2:2</value>
<value>2c0f:fa18:0:4::b</value>
<value>2606:2800:602c:b::d004</value>
<value>2610:18:181:4000::66</value>
<value>2001:48f8:1000:1::16</value>
<value>2408:8000:c000::1</value>
<value>2a03:4200:441:2::4e</value>
<value>2400:dd00:1:200a::2</value>
<value>2a02:e980:83:5b09:ecb8:c669:b336:650e</value>
<value>2001:16a0:2:200a::2</value>
<value>2001:4888:1f:e891:161:26::</value>
<value>2a0c:f743::1</value>
<value>2a02:e980:b::1</value>
<value>2001:578:201:1::601:9</value>
<value>2001:438:ffff::407d:1bc1</value>
<value>2001:920:1833::1</value>
<value>2001:1b70:a1:610::b102:2</value>
<value>2001:13c7:6014::1</value>
<value>2003:0:1203:4001::1</value>
<value>2804:a8:2:c8::d6</value>
<value>2a02:2e00:2080:f000:0:261:1:11</value>
<value>2001:578:20::d</value>
<value>2001:550:2:48::34:1</value>
<value>2a03:9d40:fe00:5::</value>
<value>2403:e800:200:102::2</value>
<!-- The CAIDA UCSD IPv4 Routed /24 DNS Names Dataset – 20171130,
http://www.caida.org/data/active/ipv4_dnsnames_dataset.xml.
Randomly selected entries from first 50000 rows of dataset.
IPv4 addresses from dataset are represented in IPv6 form. -->
<value tag="mapped">::ffff:116.253.40.133</value>
<value tag="mapped">::ffff:183.247.232.58</value>
<value tag="mapped">::ffff:116.106.34.242</value>
<value tag="mapped">::ffff:111.56.27.171</value>
<value tag="mapped">::ffff:183.245.137.140</value>
<value tag="mapped">::ffff:183.212.25.70</value>
<value tag="mapped">::ffff:162.144.2.57</value>
<value tag="mapped">::ffff:111.4.229.190</value>
<value tag="mapped">::ffff:59.52.3.168</value>
<value tag="mapped">::ffff:115.11.21.200</value>
<value tag="mapped">::ffff:121.28.97.113</value>
<value tag="mapped">::ffff:111.46.39.248</value>
<value tag="mapped">::ffff:120.192.122.34</value>
<value tag="mapped">::ffff:113.56.44.105</value>
<value tag="mapped">::ffff:116.66.238.92</value>
<value tag="mapped">::ffff:67.22.254.206</value>
<value tag="mapped">::ffff:115.0.24.191</value>
<value tag="mapped">::ffff:182.30.107.86</value>
<value tag="mapped">::ffff:223.73.153.243</value>
<value tag="mapped">::ffff:115.159.103.38</value>
<value tag="mapped">::ffff:36.186.75.121</value>
<value tag="mapped">::ffff:111.56.188.125</value>
<value tag="mapped">::ffff:115.14.93.25</value>
<value tag="mapped">::ffff:211.97.110.141</value>
<value tag="mapped">::ffff:61.58.96.173</value>
<value tag="mapped">::ffff:203.126.212.37</value>
<value tag="mapped">::ffff:192.220.125.142</value>
<value tag="mapped">::ffff:115.22.20.223</value>
<value tag="mapped">::ffff:121.25.160.80</value>
<value tag="mapped">::ffff:117.150.98.199</value>
<value tag="mapped">::ffff:183.211.172.143</value>
<value tag="mapped">::ffff:180.244.18.143</value>
<value tag="mapped">::ffff:209.131.3.252</value>
<value tag="mapped">::ffff:220.200.1.22</value>
<value tag="mapped">::ffff:171.225.130.45</value>
<value tag="mapped">::ffff:115.4.78.200</value>
<value tag="mapped">::ffff:36.183.59.29</value>
<value tag="mapped">::ffff:218.42.159.17</value>
<value tag="mapped">::ffff:115.13.39.164</value>
<value tag="mapped">::ffff:142.254.161.133</value>
<value tag="mapped">::ffff:116.2.211.43</value>
<value tag="mapped">::ffff:36.183.126.25</value>
<value tag="mapped">::ffff:66.150.171.196</value>
<value tag="mapped">::ffff:104.149.148.137</value>
<value tag="mapped">::ffff:120.239.82.212</value>
<value tag="mapped">::ffff:111.14.182.156</value>
<value tag="mapped">::ffff:115.6.63.224</value>
<value tag="mapped">::ffff:153.35.83.233</value>
<value tag="mapped">::ffff:113.142.1.1</value>
<value tag="mapped">::ffff:121.25.82.29</value>
<value tag="mapped">::ffff:62.151.203.189</value>
<value tag="mapped">::ffff:104.27.46.146</value>
<value tag="mapped">::ffff:36.189.46.88</value>
<value tag="mapped">::ffff:116.252.54.207</value>
<value tag="mapped">::ffff:64.77.240.1</value>
<value tag="mapped">::ffff:142.252.102.78</value>
<value tag="mapped">::ffff:36.82.224.170</value>
<value tag="mapped">::ffff:117.33.191.217</value>
<value tag="mapped">::ffff:144.12.164.251</value>
<value tag="mapped">::ffff:122.10.93.66</value>
<value tag="mapped">::ffff:104.25.84.59</value>
<value tag="mapped">::ffff:111.4.242.106</value>
<value tag="mapped">::ffff:222.216.51.186</value>
<value tag="mapped">::ffff:112.33.13.212</value>
<value tag="mapped">::ffff:115.9.240.116</value>
<value tag="mapped">::ffff:171.228.0.153</value>
<value tag="mapped">::ffff:45.3.47.158</value>
<value tag="mapped">::ffff:69.57.193.230</value>
<value tag="mapped">::ffff:115.6.104.199</value>
<value tag="mapped">::ffff:104.24.237.140</value>
<value tag="mapped">::ffff:199.17.84.108</value>
<value tag="mapped">::ffff:120.193.17.57</value>
<value tag="mapped">::ffff:112.40.38.145</value>
<value tag="mapped">::ffff:67.55.90.43</value>
<value tag="mapped">::ffff:180.253.57.249</value>
<value tag="mapped">::ffff:14.204.253.158</value>
<value tag="mapped">::ffff:1.83.241.116</value>
<value tag="mapped">::ffff:202.198.37.147</value>
<value tag="mapped">::ffff:115.6.31.95</value>
<value tag="mapped">::ffff:117.32.14.179</value>
<value tag="mapped">::ffff:23.238.237.26</value>
<value tag="mapped">::ffff:116.97.76.104</value>
<value tag="mapped">::ffff:1.80.2.248</value>
<value tag="mapped">::ffff:59.50.185.152</value>
<value tag="mapped">::ffff:42.117.228.166</value>
<value tag="mapped">::ffff:119.36.22.147</value>
<value tag="mapped">::ffff:210.66.18.184</value>
<value tag="mapped">::ffff:115.19.192.159</value>
<value tag="mapped">::ffff:112.15.128.113</value>
<value tag="mapped">::ffff:1.55.138.211</value>
<value tag="mapped">::ffff:210.183.19.113</value>
<value tag="mapped">::ffff:42.115.43.114</value>
<value tag="mapped">::ffff:58.16.171.31</value>
<value tag="mapped">::ffff:171.234.78.185</value>
<value tag="mapped">::ffff:113.56.43.134</value>
<value tag="mapped">::ffff:111.53.182.225</value>
<value tag="mapped">::ffff:107.160.215.141</value>
<value tag="mapped">::ffff:171.229.231.90</value>
<value tag="mapped">::ffff:58.19.84.138</value>
<value tag="mapped">::ffff:36.79.88.107</value>
<!-- invalid values -->
<value tag="error"></value>
<value tag="error"> </value>
<value tag="error">1</value>
<value tag="error">1.</value>
<value tag="error">1.2.</value>
<value tag="error">.2.</value>
<value tag="error">abc</value>
<value tag="error">ab:cd:ef:gh:ij:kl:mn</value>
<value tag="error">ffffffffffffff</value>
<value tag="error">abcdefghijklmn</value>
<value tag="error">::::::::::::::</value>
<value tag="error">::ffff:127.0.0.1 </value>
<value tag="error"> ::ffff:127.0.0.1</value>
<value tag="error">::ffff:999.999.999.999</value>
</values>
</substitution>
</substitutions>
<query tag="IPv6StringToNum">SELECT count() FROM system.numbers WHERE NOT ignore(IPv6StringToNum(materialize('{ipv6_string}')))</query>
<query tag="IPv6NumToString+IPv6StringToNum">SELECT count() FROM system.numbers WHERE NOT ignore(IPv6NumToString(IPv6StringToNum(materialize('{ipv6_string}'))))</query>
</test>

2
dbms/tests/queries/0_stateless/00076_ip_coding_functions.reference
View File

@ -16,6 +16,8 @@
1
1
1
1
1
00000000000000000000FFFF4D583737
00000000000000000000FFFF4D583737
00000000000000000000FFFF7F000001

2
dbms/tests/queries/0_stateless/00076_ip_coding_functions.sql
View File

@ -6,6 +6,8 @@ select IPv4StringToNum('127.0.0.1' as p) == (0x7f000001 as n), IPv4NumToString(n
select IPv4StringToNum(materialize('127.0.0.1') as p) == (materialize(0x7f000001) as n), IPv4NumToString(n) == p;
select IPv4NumToString(toUInt32(0)) == '0.0.0.0';
select IPv4NumToString(materialize(toUInt32(0))) == materialize('0.0.0.0');
select IPv4NumToString(toUInt32(0x7f000001)) == '127.0.0.1';
select IPv4NumToString(materialize(toUInt32(0x7f000001))) == materialize('127.0.0.1');
select IPv6NumToString(toFixedString('', 16)) == '::';
select IPv6NumToString(toFixedString(materialize(''), 16)) == materialize('::');

27
dbms/tests/queries/0_stateless/00725_ipv4_ipv6_domains.reference Normal file
View File

@ -0,0 +1,27 @@
CREATE TABLE test.ipv4_test ( ipv4_ IPv4) ENGINE = Memory
0.0.0.0 00
8.8.8.8 08080808
127.0.0.1 7F000001
192.168.0.91 C0A8005B
255.255.255.255 FFFFFFFF
< 127.0.0.1 0.0.0.0
< 127.0.0.1 8.8.8.8
> 127.0.0.1 192.168.0.91
> 127.0.0.1 255.255.255.255
= 127.0.0.1 127.0.0.1
euqality of IPv4-mapped IPv6 value and IPv4 promoted to IPv6 with function: 1
CREATE TABLE test.ipv6_test ( ipv6_ IPv6) ENGINE = Memory
:: 00000000000000000000000000000000
:: 00000000000000000000000000000000
::ffff:8.8.8.8 00000000000000000000FFFF08080808
::ffff:127.0.0.1 00000000000000000000FFFF7F000001
::ffff:193.252.17.10 00000000000000000000FFFFC1FC110A
2001:db8:ac10:fe01:feed:babe:cafe:f00d 20010DB8AC10FE01FEEDBABECAFEF00D
ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
< 127.0.0.1 ::
< 127.0.0.1 ::
< 127.0.0.1 ::ffff:8.8.8.8
> 127.0.0.1 ::ffff:193.252.17.10
> 127.0.0.1 2001:db8:ac10:fe01:feed:babe:cafe:f00d
> 127.0.0.1 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
= 127.0.0.1 ::ffff:127.0.0.1

59
dbms/tests/queries/0_stateless/00725_ipv4_ipv6_domains.sql Normal file
View File

@ -0,0 +1,59 @@
DROP TABLE IF EXISTS test.ipv4_test;
-- Only valid values for IPv4
CREATE TABLE test.ipv4_test (ipv4_ IPv4) ENGINE = Memory;
-- ipv4_ column shoud have type 'IPv4'
SHOW CREATE TABLE test.ipv4_test;
INSERT INTO test.ipv4_test (ipv4_) VALUES ('0.0.0.0'), ('255.255.255.255'), ('192.168.0.91'), ('127.0.0.1'), ('8.8.8.8');
SELECT ipv4_, hex(ipv4_) FROM test.ipv4_test ORDER BY ipv4_;
SELECT '< 127.0.0.1', ipv4_ FROM test.ipv4_test
WHERE ipv4_ < toIPv4('127.0.0.1')
ORDER BY ipv4_;
SELECT '> 127.0.0.1', ipv4_ FROM test.ipv4_test
WHERE ipv4_ > toIPv4('127.0.0.1')
ORDER BY ipv4_;
SELECT '= 127.0.0.1', ipv4_ FROM test.ipv4_test
WHERE ipv4_ = toIPv4('127.0.0.1')
ORDER BY ipv4_;
-- TODO: Assert that invalid values can't be inserted into IPv4 column.
DROP TABLE IF EXISTS test.ipv4_test;
select 'euqality of IPv4-mapped IPv6 value and IPv4 promoted to IPv6 with function:', toIPv6('::ffff:127.0.0.1') = IPv4ToIPv6(toIPv4('127.0.0.1'));
DROP TABLE IF EXISTS test.ipv6_test;
-- Only valid values for IPv6
CREATE TABLE test.ipv6_test (ipv6_ IPv6) ENGINE = Memory;
-- ipv6_ column shoud have type 'IPv6'
SHOW CREATE TABLE test.ipv6_test;
INSERT INTO test.ipv6_test VALUES ('::'), ('0:0:0:0:0:0:0:0'), ('FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'), ('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D'), ('0000:0000:0000:0000:0000:FFFF:C1FC:110A'), ('::ffff:127.0.0.1'), ('::ffff:8.8.8.8');
SELECT ipv6_, hex(ipv6_) FROM test.ipv6_test ORDER BY ipv6_;
SELECT '< 127.0.0.1', ipv6_ FROM test.ipv6_test
WHERE ipv6_ < IPv4ToIPv6(toIPv4('127.0.0.1'))
ORDER BY ipv6_;
SELECT '> 127.0.0.1', ipv6_ FROM test.ipv6_test
WHERE ipv6_ > IPv4ToIPv6(toIPv4('127.0.0.1'))
ORDER BY ipv6_;
SELECT '= 127.0.0.1', ipv6_ FROM test.ipv6_test
WHERE ipv6_ = IPv4ToIPv6(toIPv4('127.0.0.1'))
ORDER BY ipv6_;
-- TODO: Assert that invalid values can't be inserted into IPv6 column.
DROP TABLE IF EXISTS test.ipv6_test;