mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge pull request #6132 from vitlibar/allow-protobuf-message-with-all-fields-by-default
Allow protobuf message with all fields by default
This commit is contained in:
alexey-milovidov
GitHub
commit
20a8914858
@ -462,7 +462,7 @@ void DataTypeArray::deserializeProtobuf(IColumn & column, ProtobufReader & proto
|
||||
bool nested_row_added;
|
||||
do
|
||||
nested->deserializeProtobuf(nested_column, protobuf, true, nested_row_added);
|
||||
while (nested_row_added && protobuf.maybeCanReadValue());
|
||||
while (nested_row_added && protobuf.canReadMoreValues());
|
||||
if (allow_add_row)
|
||||
{
|
||||
offsets.emplace_back(nested_column.size());
|
||||
|
||||
@ -34,19 +34,18 @@ namespace
|
||||
BITS32 = 5,
|
||||
};
|
||||
|
||||
// The following should be always true:
|
||||
// REACHED_END < any cursor position < min(END_OF_VARINT, END_OF_GROUP)
|
||||
// The following condition must always be true:
|
||||
// any_cursor_position < min(END_OF_VARINT, END_OF_GROUP)
|
||||
// This inequation helps to check conditions in SimpleReader.
|
||||
constexpr UInt64 END_OF_VARINT = static_cast<UInt64>(-1);
|
||||
constexpr UInt64 END_OF_GROUP = static_cast<UInt64>(-2);
|
||||
|
||||
Int64 decodeZigZag(UInt64 n) { return static_cast<Int64>((n >> 1) ^ (~(n & 1) + 1)); }
|
||||
}
|
||||
|
||||
|
||||
[[noreturn]] void ProtobufReader::SimpleReader::throwUnknownFormat()
|
||||
{
|
||||
throw Exception("Protobuf messages are corrupted or don't match the provided schema. Please note that Protobuf stream is length-delimited: every message is prefixed by its length in varint.", ErrorCodes::UNKNOWN_PROTOBUF_FORMAT);
|
||||
[[noreturn]] void throwUnknownFormat()
|
||||
{
|
||||
throw Exception("Protobuf messages are corrupted or don't match the provided schema. Please note that Protobuf stream is length-delimited: every message is prefixed by its length in varint.", ErrorCodes::UNKNOWN_PROTOBUF_FORMAT);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -54,94 +53,103 @@ namespace
|
||||
// Knows nothing about protobuf schemas, just provides useful functions to deserialize data.
|
||||
ProtobufReader::SimpleReader::SimpleReader(ReadBuffer & in_)
|
||||
: in(in_)
|
||||
, cursor(1 /* We starts at cursor == 1 to keep any cursor value > REACHED_END, this allows to simplify conditions */)
|
||||
, current_message_end(REACHED_END)
|
||||
, field_end(REACHED_END)
|
||||
, cursor(0)
|
||||
, current_message_level(0)
|
||||
, current_message_end(0)
|
||||
, field_end(0)
|
||||
, last_string_pos(-1)
|
||||
{
|
||||
}
|
||||
|
||||
bool ProtobufReader::SimpleReader::startMessage()
|
||||
{
|
||||
if ((current_message_end == REACHED_END) && parent_message_ends.empty())
|
||||
{
|
||||
// Start reading a root message.
|
||||
if (unlikely(in.eof()))
|
||||
return false;
|
||||
size_t size_of_message = readVarint();
|
||||
if (size_of_message == 0)
|
||||
throwUnknownFormat();
|
||||
current_message_end = cursor + size_of_message;
|
||||
root_message_end = current_message_end;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Start reading a nested message which is located inside a length-delimited field
|
||||
// of another message.s
|
||||
parent_message_ends.emplace_back(current_message_end);
|
||||
current_message_end = field_end;
|
||||
}
|
||||
field_end = REACHED_END;
|
||||
// Start reading a root message.
|
||||
assert(!current_message_level);
|
||||
if (unlikely(in.eof()))
|
||||
return false;
|
||||
size_t size_of_message = readVarint();
|
||||
current_message_end = cursor + size_of_message;
|
||||
++current_message_level;
|
||||
field_end = cursor;
|
||||
return true;
|
||||
}
|
||||
|
||||
void ProtobufReader::SimpleReader::endMessage()
|
||||
void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
|
||||
{
|
||||
if (current_message_end != REACHED_END)
|
||||
if (!current_message_level)
|
||||
return;
|
||||
|
||||
UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
|
||||
if (cursor != root_message_end)
|
||||
{
|
||||
if (current_message_end == END_OF_GROUP)
|
||||
ignoreGroup();
|
||||
else if (cursor < current_message_end)
|
||||
ignore(current_message_end - cursor);
|
||||
else if (unlikely(cursor > current_message_end))
|
||||
{
|
||||
if (!parent_message_ends.empty())
|
||||
throwUnknownFormat();
|
||||
moveCursorBackward(cursor - current_message_end);
|
||||
}
|
||||
current_message_end = REACHED_END;
|
||||
if (cursor < root_message_end)
|
||||
ignore(root_message_end - cursor);
|
||||
else if (ignore_errors)
|
||||
moveCursorBackward(cursor - root_message_end);
|
||||
else
|
||||
throwUnknownFormat();
|
||||
}
|
||||
|
||||
field_end = REACHED_END;
|
||||
if (!parent_message_ends.empty())
|
||||
{
|
||||
current_message_end = parent_message_ends.back();
|
||||
parent_message_ends.pop_back();
|
||||
}
|
||||
current_message_level = 0;
|
||||
parent_message_ends.clear();
|
||||
}
|
||||
|
||||
void ProtobufReader::SimpleReader::endRootMessage()
|
||||
void ProtobufReader::SimpleReader::startNestedMessage()
|
||||
{
|
||||
UInt64 message_end = parent_message_ends.empty() ? current_message_end : parent_message_ends.front();
|
||||
if (message_end != REACHED_END)
|
||||
assert(current_message_level >= 1);
|
||||
// Start reading a nested message which is located inside a length-delimited field
|
||||
// of another message.
|
||||
parent_message_ends.emplace_back(current_message_end);
|
||||
current_message_end = field_end;
|
||||
++current_message_level;
|
||||
field_end = cursor;
|
||||
}
|
||||
|
||||
void ProtobufReader::SimpleReader::endNestedMessage()
|
||||
{
|
||||
assert(current_message_level >= 2);
|
||||
if (cursor != current_message_end)
|
||||
{
|
||||
if (cursor < message_end)
|
||||
ignore(message_end - cursor);
|
||||
else if (unlikely(cursor > message_end))
|
||||
moveCursorBackward(cursor - message_end);
|
||||
if (current_message_end == END_OF_GROUP)
|
||||
{
|
||||
ignoreGroup();
|
||||
current_message_end = cursor;
|
||||
}
|
||||
else if (cursor < current_message_end)
|
||||
ignore(current_message_end - cursor);
|
||||
else
|
||||
throwUnknownFormat();
|
||||
}
|
||||
parent_message_ends.clear();
|
||||
current_message_end = REACHED_END;
|
||||
field_end = REACHED_END;
|
||||
|
||||
--current_message_level;
|
||||
current_message_end = parent_message_ends.back();
|
||||
parent_message_ends.pop_back();
|
||||
field_end = cursor;
|
||||
}
|
||||
|
||||
bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
|
||||
{
|
||||
if (field_end != REACHED_END)
|
||||
assert(current_message_level);
|
||||
if (field_end != cursor)
|
||||
{
|
||||
if (field_end == END_OF_VARINT)
|
||||
{
|
||||
ignoreVarint();
|
||||
field_end = cursor;
|
||||
}
|
||||
else if (field_end == END_OF_GROUP)
|
||||
{
|
||||
ignoreGroup();
|
||||
field_end = cursor;
|
||||
}
|
||||
else if (cursor < field_end)
|
||||
ignore(field_end - cursor);
|
||||
field_end = REACHED_END;
|
||||
else
|
||||
throwUnknownFormat();
|
||||
}
|
||||
|
||||
if (cursor >= current_message_end)
|
||||
{
|
||||
current_message_end = REACHED_END;
|
||||
return false;
|
||||
}
|
||||
|
||||
UInt64 varint = readVarint();
|
||||
if (unlikely(varint & (static_cast<UInt64>(0xFFFFFFFF) << 32)))
|
||||
@ -151,6 +159,11 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
|
||||
WireType wire_type = static_cast<WireType>(key & 0x07);
|
||||
switch (wire_type)
|
||||
{
|
||||
case BITS32:
|
||||
{
|
||||
field_end = cursor + 4;
|
||||
return true;
|
||||
}
|
||||
case BITS64:
|
||||
{
|
||||
field_end = cursor + 8;
|
||||
@ -176,29 +189,20 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
|
||||
{
|
||||
if (current_message_end != END_OF_GROUP)
|
||||
throwUnknownFormat();
|
||||
current_message_end = REACHED_END;
|
||||
current_message_end = cursor;
|
||||
return false;
|
||||
}
|
||||
case BITS32:
|
||||
{
|
||||
field_end = cursor + 4;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
throwUnknownFormat();
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
bool ProtobufReader::SimpleReader::readUInt(UInt64 & value)
|
||||
{
|
||||
if (unlikely(cursor >= field_end))
|
||||
{
|
||||
field_end = REACHED_END;
|
||||
return false;
|
||||
}
|
||||
value = readVarint();
|
||||
if ((field_end == END_OF_VARINT) || (cursor >= field_end))
|
||||
field_end = REACHED_END;
|
||||
if (field_end == END_OF_VARINT)
|
||||
field_end = cursor;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -224,25 +228,22 @@ template<typename T>
|
||||
bool ProtobufReader::SimpleReader::readFixed(T & value)
|
||||
{
|
||||
if (unlikely(cursor >= field_end))
|
||||
{
|
||||
field_end = REACHED_END;
|
||||
return false;
|
||||
}
|
||||
readBinary(&value, sizeof(T));
|
||||
if (cursor >= field_end)
|
||||
field_end = REACHED_END;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ProtobufReader::SimpleReader::readStringInto(PaddedPODArray<UInt8> & str)
|
||||
{
|
||||
if (unlikely(cursor == last_string_pos))
|
||||
return false; /// We don't want to read the same empty string again.
|
||||
last_string_pos = cursor;
|
||||
if (unlikely(cursor > field_end))
|
||||
return false;
|
||||
throwUnknownFormat();
|
||||
size_t length = field_end - cursor;
|
||||
size_t old_size = str.size();
|
||||
str.resize(old_size + length);
|
||||
readBinary(reinterpret_cast<char*>(str.data() + old_size), length);
|
||||
field_end = REACHED_END;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -299,7 +300,6 @@ UInt64 ProtobufReader::SimpleReader::continueReadingVarint(UInt64 first_byte)
|
||||
#undef PROTOBUF_READER_READ_VARINT_BYTE
|
||||
|
||||
throwUnknownFormat();
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
void ProtobufReader::SimpleReader::ignoreVarint()
|
||||
@ -1083,9 +1083,9 @@ bool ProtobufReader::startMessage()
|
||||
return true;
|
||||
}
|
||||
|
||||
void ProtobufReader::endMessage()
|
||||
void ProtobufReader::endMessage(bool try_ignore_errors)
|
||||
{
|
||||
simple_reader.endRootMessage();
|
||||
simple_reader.endMessage(try_ignore_errors);
|
||||
current_message = nullptr;
|
||||
current_converter = nullptr;
|
||||
}
|
||||
@ -1102,7 +1102,7 @@ bool ProtobufReader::readColumnIndex(size_t & column_index)
|
||||
current_converter = nullptr;
|
||||
return false;
|
||||
}
|
||||
simple_reader.endMessage();
|
||||
simple_reader.endNestedMessage();
|
||||
current_field_index = current_message->index_in_parent;
|
||||
current_message = current_message->parent;
|
||||
continue;
|
||||
@ -1132,7 +1132,7 @@ bool ProtobufReader::readColumnIndex(size_t & column_index)
|
||||
|
||||
if (field->nested_message)
|
||||
{
|
||||
simple_reader.startMessage();
|
||||
simple_reader.startNestedMessage();
|
||||
current_message = field->nested_message.get();
|
||||
current_field_index = 0;
|
||||
continue;
|
||||
|
||||
@ -42,7 +42,7 @@ public:
|
||||
bool startMessage();
|
||||
|
||||
/// Ends reading a message.
|
||||
void endMessage();
|
||||
void endMessage(bool ignore_errors = false);
|
||||
|
||||
/// Reads the column index.
|
||||
/// The function returns false if there are no more columns to read (call endMessage() in this case).
|
||||
@ -79,9 +79,8 @@ public:
|
||||
|
||||
bool readAggregateFunction(const AggregateFunctionPtr & function, AggregateDataPtr place, Arena & arena) { return current_converter->readAggregateFunction(function, place, arena); }
|
||||
|
||||
/// When it returns false there is no more values left and from now on all the read<Type>() functions will return false
|
||||
/// until readColumnIndex() is called. When it returns true it's unclear.
|
||||
bool ALWAYS_INLINE maybeCanReadValue() const { return simple_reader.maybeCanReadValue(); }
|
||||
/// Call it after calling one of the read*() function to determine if there are more values available for reading.
|
||||
bool ALWAYS_INLINE canReadMoreValues() const { return simple_reader.canReadMoreValues(); }
|
||||
|
||||
private:
|
||||
class SimpleReader
|
||||
@ -89,8 +88,9 @@ private:
|
||||
public:
|
||||
SimpleReader(ReadBuffer & in_);
|
||||
bool startMessage();
|
||||
void endMessage();
|
||||
void endRootMessage();
|
||||
void endMessage(bool ignore_errors);
|
||||
void startNestedMessage();
|
||||
void endNestedMessage();
|
||||
bool readFieldNumber(UInt32 & field_number);
|
||||
bool readInt(Int64 & value);
|
||||
bool readSInt(Int64 & value);
|
||||
@ -98,15 +98,7 @@ private:
|
||||
template<typename T> bool readFixed(T & value);
|
||||
bool readStringInto(PaddedPODArray<UInt8> & str);
|
||||
|
||||
bool ALWAYS_INLINE maybeCanReadValue() const
|
||||
{
|
||||
if (field_end == REACHED_END)
|
||||
return false;
|
||||
if (cursor < root_message_end)
|
||||
return true;
|
||||
|
||||
throwUnknownFormat();
|
||||
}
|
||||
bool ALWAYS_INLINE canReadMoreValues() const { return cursor < field_end; }
|
||||
|
||||
private:
|
||||
void readBinary(void * data, size_t size);
|
||||
@ -128,17 +120,13 @@ private:
|
||||
void ignoreVarint();
|
||||
void ignoreGroup();
|
||||
|
||||
[[noreturn]] static void throwUnknownFormat();
|
||||
|
||||
static constexpr UInt64 REACHED_END = 0;
|
||||
|
||||
ReadBuffer & in;
|
||||
UInt64 cursor;
|
||||
std::vector<UInt64> parent_message_ends;
|
||||
size_t current_message_level;
|
||||
UInt64 current_message_end;
|
||||
std::vector<UInt64> parent_message_ends;
|
||||
UInt64 field_end;
|
||||
|
||||
UInt64 root_message_end;
|
||||
UInt64 last_string_pos;
|
||||
};
|
||||
|
||||
class IConverter
|
||||
|
||||
@ -43,7 +43,7 @@ bool ProtobufRowInputStream::read(MutableColumns & columns, RowReadExtension & e
|
||||
read_columns[column_index] = true;
|
||||
allow_add_row = false;
|
||||
}
|
||||
} while (reader.maybeCanReadValue());
|
||||
} while (reader.canReadMoreValues());
|
||||
}
|
||||
|
||||
// Fill non-visited columns with the default values.
|
||||
@ -62,7 +62,7 @@ bool ProtobufRowInputStream::allowSyncAfterError() const
|
||||
|
||||
void ProtobufRowInputStream::syncAfterError()
|
||||
{
|
||||
reader.endMessage();
|
||||
reader.endMessage(true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -43,7 +43,7 @@ bool ProtobufRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
|
||||
read_columns[column_index] = true;
|
||||
allow_add_row = false;
|
||||
}
|
||||
} while (reader.maybeCanReadValue());
|
||||
} while (reader.canReadMoreValues());
|
||||
}
|
||||
|
||||
// Fill non-visited columns with the default values.
|
||||
@ -62,7 +62,7 @@ bool ProtobufRowInputFormat::allowSyncAfterError() const
|
||||
|
||||
void ProtobufRowInputFormat::syncAfterError()
|
||||
{
|
||||
reader.endMessage();
|
||||
reader.endMessage(true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -143,3 +143,9 @@ message StrPerson {
|
||||
MeasureUnits measureUnits = 21;
|
||||
NestinessA nestiness_a = 22;
|
||||
};
|
||||
|
||||
message NumberAndSquare
|
||||
{
|
||||
uint32 number = 1;
|
||||
uint64 square = 2;
|
||||
};
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
echo -ne '\xf3\x01\x0a\x24\x61\x37\x35\x32\x32\x31\x35\x38\x2d\x33\x64\x34\x31\x2d\x34\x62\x37\x37\x2d\x61\x64\x36\x39\x2d\x36\x63\x35\x39\x38\x65\x65\x35\x35\x63\x34\x39\x12\x04\x49\x76\x61\x6e\x1a\x06\x50\x65\x74\x72\x6f\x76\x20\x01\x28\xaf\x1f\x32\x03\x70\x6e\x67\x3a\x0c\x2b\x37\x34\x39\x35\x31\x32\x33\x34\x35\x36\x37\x40\x01\x4d\xfc\xd0\x30\x5c\x50\x26\x58\x09\x62\x09\x59\x65\x73\x74\x65\x72\x64\x61\x79\x62\x07\x46\x6c\x6f\x77\x65\x72\x73\x6a\x04\xff\x01\x00\x00\x72\x06\x4d\x6f\x73\x63\x6f\x77\x7a\x08\x4b\x03\x5f\x42\x72\x7d\x16\x42\x81\x01\x1f\x85\xeb\x51\xb8\x1e\x09\x40\x89\x01\x33\x33\x33\x33\x33\xc3\x6a\x40\x95\x01\xcd\xcc\xcc\x3d\x9d\x01\x9a\x99\xb9\x40\xa0\x01\x80\xc4\xd7\x8d\x7f\xaa\x01\x0c\x0a\x05\x6d\x65\x74\x65\x72\x15\x00\x00\x80\x3f\xaa\x01\x11\x0a\x0a\x63\x65\x6e\x74\x69\x6d\x65\x74\x65\x72\x15\x0a\xd7\x23\x3c\xaa\x01\x10\x0a\x09\x6b\x69\x6c\x6f\x6d\x65\x74\x65\x72\x15\x00\x00\x7a\x44\xb2\x01\x10\x0a\x0e\xa2\x06\x0b\x0a\x09\x08\xf4\x03\x12\x04\xf5\x03\xf6\x03\x7e\x0a\x24\x63\x36\x39\x34\x61\x64\x38\x61\x2d\x66\x37\x31\x34\x2d\x34\x65\x61\x33\x2d\x39\x30\x37\x64\x2d\x66\x64\x35\x34\x66\x62\x32\x35\x64\x39\x62\x35\x12\x07\x4e\x61\x74\x61\x6c\x69\x61\x1a\x08\x53\x6f\x6b\x6f\x6c\x6f\x76\x61\x28\xa6\x3f\x32\x03\x6a\x70\x67\x50\x1a\x58\x0b\x6a\x04\x64\xc8\x01\x32\x72\x08\x50\x6c\x79\x6d\x6f\x75\x74\x68\x7a\x08\x6a\x9d\x49\x42\x46\x8c\x84\xc0\x81\x01\x6e\x86\x1b\xf0\xf9\x21\x09\x40\x95\x01\x42\x60\xe5\x3b\x9d\x01\xcd\xcc\xac\x40\xa0\x01\xff\xff\xa9\xce\x93\x8c\x09\xc0\x01\x0a\x24\x61\x37\x64\x61\x31\x61\x61\x36\x2d\x66\x34\x32\x35\x2d\x34\x37\x38\x39\x2d\x38\x39\x34\x37\x2d\x62\x30\x33\x34\x37\x38\x36\x65\x64\x33\x37\x34\x12\x06\x56\x61\x73\x69\x6c\x79\x1a\x07\x53\x69\x64\x6f\x72\x6f\x76\x20\x01\x28\xfb\x48\x32\x03\x62\x6d\x70\x3a\x0d\x2b\x34\x34\x32\x30\x31\x32\x33\x34\x35\x36\x37\x38\x40\x01\x4d\x50\xe0\x27\x5c\x50\x17\x58\x04\x62\x05\x53\x75\x6e\x6e\x79\x6a\x05\xfa\x01\xf4\x01\x0a\x72\x08\x4d\x75\x72\x6d\x61\x6e\x73\x6b\x7a\x08\xfd\xf0\x89\x42\xc8\x4c\x04\x42\x81\x01\x11\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x37\x42\x95\x01\x00\x00\x48\x44\x9d\x01\xcd\xcc\x4c\xc0\xa0\x01\x80\xd4\x9f\x93\x01\xaa\x01\x0c\x0a\x05\x70\x6f\x75\x6e\x64\x15\x00\x00\x80\x41\xb2\x01\x0a\x0a\x08\xa2\x06\x05\x0a\x03\x08\xf7\x03' | $CLICKHOUSE_CLIENT --query="INSERT INTO table_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'"
|
||||
echo -ne '\xb3\x01\x12\x05\x46\x72\x69\x64\x61\x28\x99\xe1\xf3\xd1\x0b\x52\x08\x45\x72\x6d\x61\x6b\x6f\x76\x61\x72\x0c\x00\x00\xdc\x42\x00\x00\x52\x43\x00\x00\x94\x42\x79\x48\xce\x3d\x51\x00\x00\x00\x00\xc8\x02\x14\xc2\x05\x08\x00\x00\x80\x44\x00\x00\x80\x49\x9a\x06\x02\x4b\x42\x9a\x06\x02\x4d\x42\xa1\x06\x00\x00\x00\x00\x00\x00\xe0\x3f\xa8\x06\x2a\xa8\x06\xa8\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\x25\x06\x49\x40\xfa\x06\x02\x34\x30\x90\x08\xe2\x08\xe1\x08\x89\xe6\x6e\xdd\x01\x00\x00\x00\xb0\x09\xc3\x19\xd0\x0c\xb7\x02\xe2\x12\x24\x32\x30\x66\x63\x64\x39\x35\x61\x2d\x33\x33\x32\x64\x2d\x34\x31\x64\x62\x2d\x61\x39\x65\x63\x2d\x31\x36\x31\x66\x36\x34\x34\x64\x30\x35\x39\x63\xa0\x38\xbc\x05\xaa\x38\x02\xbd\x05\xb4\x01\x08\x01\x12\x06\x49\x73\x6f\x6c\x64\x65\x52\x07\x4c\x61\x76\x72\x6f\x76\x61\x72\x0c\x00\x00\x7f\x43\x00\x00\x00\x00\x00\x00\x7f\x43\xaa\x01\x03\x61\x62\x63\xc8\x02\x32\xc2\x05\x08\x00\x00\x00\x41\x00\x00\x80\x3f\x9a\x06\x04\x42\x79\x74\x65\x9a\x06\x03\x42\x69\x74\xa1\x06\x00\x00\x00\x00\x00\x00\x12\x40\xa8\x06\x1a\xa8\x06\xb0\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\xf9\x0f\x49\x40\xc2\x06\x01\x2c\xfa\x06\x02\x33\x32\x90\x08\x78\xe1\x08\x39\x4e\x2b\xfe\xe4\xf5\xff\xff\xb0\x09\xe8\x30\xd8\x12\x01\xe2\x12\x24\x37\x63\x66\x61\x36\x38\x35\x36\x2d\x61\x35\x34\x61\x2d\x34\x37\x38\x36\x2d\x62\x38\x65\x35\x2d\x37\x34\x35\x31\x35\x39\x64\x35\x32\x32\x37\x38\xa0\x38\xbe\x05\xc2\x3e\x05\x15\x00\x00\xb6\x42' | $CLICKHOUSE_CLIENT --query="INSERT INTO table_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson'"
|
||||
echo -ne '\xa5\x02\x0a\x24\x61\x61\x30\x65\x35\x61\x30\x36\x2d\x63\x61\x62\x32\x2d\x34\x30\x33\x34\x2d\x61\x36\x61\x32\x2d\x34\x38\x65\x38\x32\x62\x39\x31\x36\x36\x34\x65\x12\x06\x4c\x65\x6f\x6e\x69\x64\x1a\x08\x4b\x69\x72\x69\x6c\x6c\x6f\x76\x22\x04\x6d\x61\x6c\x65\x2a\x0a\x31\x39\x38\x33\x2d\x30\x36\x2d\x32\x34\x3a\x0c\x2b\x37\x34\x39\x35\x30\x32\x37\x35\x38\x36\x34\x42\x01\x31\x4a\x13\x32\x30\x31\x39\x2d\x30\x32\x2d\x30\x34\x20\x30\x39\x3a\x34\x35\x3a\x30\x30\x52\x02\x33\x35\x5a\x06\x63\x61\x6e\x63\x65\x72\x62\x07\x37\x20\x72\x69\x6e\x67\x73\x62\x08\x45\x61\x73\x74\x73\x69\x64\x65\x62\x0b\x4c\x61\x73\x74\x20\x48\x75\x72\x72\x61\x68\x6a\x01\x30\x6a\x01\x30\x6a\x03\x32\x35\x35\x72\x09\x53\x61\x6e\x20\x44\x69\x65\x67\x6f\x7a\x09\x33\x32\x2e\x38\x32\x33\x39\x34\x33\x7a\x0b\x2d\x31\x31\x37\x2e\x30\x38\x31\x33\x32\x37\x82\x01\x09\x33\x2e\x31\x34\x31\x35\x39\x32\x37\x8a\x01\x08\x31\x35\x30\x30\x30\x30\x30\x30\x92\x01\x06\x31\x38\x36\x2e\x37\x35\x9a\x01\x04\x2d\x32\x2e\x31\xa2\x01\x0b\x32\x30\x36\x35\x39\x38\x32\x39\x33\x33\x31\xaa\x01\x18\x0a\x06\x6d\x69\x6e\x75\x74\x65\x0a\x04\x68\x6f\x75\x72\x12\x02\x36\x30\x12\x04\x33\x36\x30\x30\xb2\x01\x08\x0a\x06\x12\x04\x31\x38\x30\x30' | $CLICKHOUSE_CLIENT --query="INSERT INTO table_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson'"
|
||||
echo -ne '\xdd\x01\x0a\x24\x33\x66\x61\x65\x65\x30\x36\x34\x2d\x63\x34\x66\x37\x2d\x34\x64\x33\x34\x2d\x62\x36\x66\x33\x2d\x38\x64\x38\x31\x63\x32\x62\x36\x61\x31\x35\x64\x12\x04\x4e\x69\x63\x6b\x1a\x0a\x4b\x6f\x6c\x65\x73\x6e\x69\x6b\x6f\x76\x20\x01\x28\xda\x52\x32\x03\x62\x6d\x70\x3a\x0c\x34\x31\x32\x2d\x36\x38\x37\x2d\x35\x30\x30\x37\x40\x01\x4d\x2f\x27\xf2\x5b\x50\x14\x58\x09\x62\x06\x48\x61\x76\x61\x6e\x61\x68\x80\x01\x68\x00\x68\x80\x01\x72\x0a\x50\x69\x74\x74\x73\x62\x75\x72\x67\x68\x7a\x08\x9b\x11\x22\x42\x1f\xe6\x9f\xc2\x81\x01\x28\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x27\x42\x95\x01\x00\x00\x43\x44\x9d\x01\x66\x66\x92\x41\xa0\x01\xce\xdf\xb8\xba\x01\xab\x01\x0d\xcd\xcc\xe2\x41\x0d\xcd\xcc\x4c\x3e\x0d\x00\x00\x80\x3f\x12\x05\x6f\x75\x6e\x63\x65\x12\x05\x63\x61\x72\x61\x74\x12\x04\x67\x72\x61\x6d\xac\x01\xb3\x01\x0b\xa2\x06\x05\x0b\x08\x96\x4a\x0c\x0c\xb4\x01' | $CLICKHOUSE_CLIENT --query="INSERT INTO table_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'"
|
||||
echo -ne '\xf3\x01\x0a\x24\x61\x37\x35\x32\x32\x31\x35\x38\x2d\x33\x64\x34\x31\x2d\x34\x62\x37\x37\x2d\x61\x64\x36\x39\x2d\x36\x63\x35\x39\x38\x65\x65\x35\x35\x63\x34\x39\x12\x04\x49\x76\x61\x6e\x1a\x06\x50\x65\x74\x72\x6f\x76\x20\x01\x28\xaf\x1f\x32\x03\x70\x6e\x67\x3a\x0c\x2b\x37\x34\x39\x35\x31\x32\x33\x34\x35\x36\x37\x40\x01\x4d\xfc\xd0\x30\x5c\x50\x26\x58\x09\x62\x09\x59\x65\x73\x74\x65\x72\x64\x61\x79\x62\x07\x46\x6c\x6f\x77\x65\x72\x73\x6a\x04\xff\x01\x00\x00\x72\x06\x4d\x6f\x73\x63\x6f\x77\x7a\x08\x4b\x03\x5f\x42\x72\x7d\x16\x42\x81\x01\x1f\x85\xeb\x51\xb8\x1e\x09\x40\x89\x01\x33\x33\x33\x33\x33\xc3\x6a\x40\x95\x01\xcd\xcc\xcc\x3d\x9d\x01\x9a\x99\xb9\x40\xa0\x01\x80\xc4\xd7\x8d\x7f\xaa\x01\x0c\x0a\x05\x6d\x65\x74\x65\x72\x15\x00\x00\x80\x3f\xaa\x01\x11\x0a\x0a\x63\x65\x6e\x74\x69\x6d\x65\x74\x65\x72\x15\x0a\xd7\x23\x3c\xaa\x01\x10\x0a\x09\x6b\x69\x6c\x6f\x6d\x65\x74\x65\x72\x15\x00\x00\x7a\x44\xb2\x01\x10\x0a\x0e\xa2\x06\x0b\x0a\x09\x08\xf4\x03\x12\x04\xf5\x03\xf6\x03\x7e\x0a\x24\x63\x36\x39\x34\x61\x64\x38\x61\x2d\x66\x37\x31\x34\x2d\x34\x65\x61\x33\x2d\x39\x30\x37\x64\x2d\x66\x64\x35\x34\x66\x62\x32\x35\x64\x39\x62\x35\x12\x07\x4e\x61\x74\x61\x6c\x69\x61\x1a\x08\x53\x6f\x6b\x6f\x6c\x6f\x76\x61\x28\xa6\x3f\x32\x03\x6a\x70\x67\x50\x1a\x58\x0b\x6a\x04\x64\xc8\x01\x32\x72\x08\x50\x6c\x79\x6d\x6f\x75\x74\x68\x7a\x08\x6a\x9d\x49\x42\x46\x8c\x84\xc0\x81\x01\x6e\x86\x1b\xf0\xf9\x21\x09\x40\x95\x01\x42\x60\xe5\x3b\x9d\x01\xcd\xcc\xac\x40\xa0\x01\xff\xff\xa9\xce\x93\x8c\x09\xc0\x01\x0a\x24\x61\x37\x64\x61\x31\x61\x61\x36\x2d\x66\x34\x32\x35\x2d\x34\x37\x38\x39\x2d\x38\x39\x34\x37\x2d\x62\x30\x33\x34\x37\x38\x36\x65\x64\x33\x37\x34\x12\x06\x56\x61\x73\x69\x6c\x79\x1a\x07\x53\x69\x64\x6f\x72\x6f\x76\x20\x01\x28\xfb\x48\x32\x03\x62\x6d\x70\x3a\x0d\x2b\x34\x34\x32\x30\x31\x32\x33\x34\x35\x36\x37\x38\x40\x01\x4d\x50\xe0\x27\x5c\x50\x17\x58\x04\x62\x05\x53\x75\x6e\x6e\x79\x6a\x05\xfa\x01\xf4\x01\x0a\x72\x08\x4d\x75\x72\x6d\x61\x6e\x73\x6b\x7a\x08\xfd\xf0\x89\x42\xc8\x4c\x04\x42\x81\x01\x11\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x37\x42\x95\x01\x00\x00\x48\x44\x9d\x01\xcd\xcc\x4c\xc0\xa0\x01\x80\xd4\x9f\x93\x01\xaa\x01\x0c\x0a\x05\x70\x6f\x75\x6e\x64\x15\x00\x00\x80\x41\xb2\x01\x0a\x0a\x08\xa2\x06\x05\x0a\x03\x08\xf7\x03' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'"
|
||||
echo -ne '\xb3\x01\x12\x05\x46\x72\x69\x64\x61\x28\x99\xe1\xf3\xd1\x0b\x52\x08\x45\x72\x6d\x61\x6b\x6f\x76\x61\x72\x0c\x00\x00\xdc\x42\x00\x00\x52\x43\x00\x00\x94\x42\x79\x48\xce\x3d\x51\x00\x00\x00\x00\xc8\x02\x14\xc2\x05\x08\x00\x00\x80\x44\x00\x00\x80\x49\x9a\x06\x02\x4b\x42\x9a\x06\x02\x4d\x42\xa1\x06\x00\x00\x00\x00\x00\x00\xe0\x3f\xa8\x06\x2a\xa8\x06\xa8\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\x25\x06\x49\x40\xfa\x06\x02\x34\x30\x90\x08\xe2\x08\xe1\x08\x89\xe6\x6e\xdd\x01\x00\x00\x00\xb0\x09\xc3\x19\xd0\x0c\xb7\x02\xe2\x12\x24\x32\x30\x66\x63\x64\x39\x35\x61\x2d\x33\x33\x32\x64\x2d\x34\x31\x64\x62\x2d\x61\x39\x65\x63\x2d\x31\x36\x31\x66\x36\x34\x34\x64\x30\x35\x39\x63\xa0\x38\xbc\x05\xaa\x38\x02\xbd\x05\xb4\x01\x08\x01\x12\x06\x49\x73\x6f\x6c\x64\x65\x52\x07\x4c\x61\x76\x72\x6f\x76\x61\x72\x0c\x00\x00\x7f\x43\x00\x00\x00\x00\x00\x00\x7f\x43\xaa\x01\x03\x61\x62\x63\xc8\x02\x32\xc2\x05\x08\x00\x00\x00\x41\x00\x00\x80\x3f\x9a\x06\x04\x42\x79\x74\x65\x9a\x06\x03\x42\x69\x74\xa1\x06\x00\x00\x00\x00\x00\x00\x12\x40\xa8\x06\x1a\xa8\x06\xb0\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\xf9\x0f\x49\x40\xc2\x06\x01\x2c\xfa\x06\x02\x33\x32\x90\x08\x78\xe1\x08\x39\x4e\x2b\xfe\xe4\xf5\xff\xff\xb0\x09\xe8\x30\xd8\x12\x01\xe2\x12\x24\x37\x63\x66\x61\x36\x38\x35\x36\x2d\x61\x35\x34\x61\x2d\x34\x37\x38\x36\x2d\x62\x38\x65\x35\x2d\x37\x34\x35\x31\x35\x39\x64\x35\x32\x32\x37\x38\xa0\x38\xbe\x05\xc2\x3e\x05\x15\x00\x00\xb6\x42' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson'"
|
||||
echo -ne '\xa5\x02\x0a\x24\x61\x61\x30\x65\x35\x61\x30\x36\x2d\x63\x61\x62\x32\x2d\x34\x30\x33\x34\x2d\x61\x36\x61\x32\x2d\x34\x38\x65\x38\x32\x62\x39\x31\x36\x36\x34\x65\x12\x06\x4c\x65\x6f\x6e\x69\x64\x1a\x08\x4b\x69\x72\x69\x6c\x6c\x6f\x76\x22\x04\x6d\x61\x6c\x65\x2a\x0a\x31\x39\x38\x33\x2d\x30\x36\x2d\x32\x34\x3a\x0c\x2b\x37\x34\x39\x35\x30\x32\x37\x35\x38\x36\x34\x42\x01\x31\x4a\x13\x32\x30\x31\x39\x2d\x30\x32\x2d\x30\x34\x20\x30\x39\x3a\x34\x35\x3a\x30\x30\x52\x02\x33\x35\x5a\x06\x63\x61\x6e\x63\x65\x72\x62\x07\x37\x20\x72\x69\x6e\x67\x73\x62\x08\x45\x61\x73\x74\x73\x69\x64\x65\x62\x0b\x4c\x61\x73\x74\x20\x48\x75\x72\x72\x61\x68\x6a\x01\x30\x6a\x01\x30\x6a\x03\x32\x35\x35\x72\x09\x53\x61\x6e\x20\x44\x69\x65\x67\x6f\x7a\x09\x33\x32\x2e\x38\x32\x33\x39\x34\x33\x7a\x0b\x2d\x31\x31\x37\x2e\x30\x38\x31\x33\x32\x37\x82\x01\x09\x33\x2e\x31\x34\x31\x35\x39\x32\x37\x8a\x01\x08\x31\x35\x30\x30\x30\x30\x30\x30\x92\x01\x06\x31\x38\x36\x2e\x37\x35\x9a\x01\x04\x2d\x32\x2e\x31\xa2\x01\x0b\x32\x30\x36\x35\x39\x38\x32\x39\x33\x33\x31\xaa\x01\x18\x0a\x06\x6d\x69\x6e\x75\x74\x65\x0a\x04\x68\x6f\x75\x72\x12\x02\x36\x30\x12\x04\x33\x36\x30\x30\xb2\x01\x08\x0a\x06\x12\x04\x31\x38\x30\x30' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson'"
|
||||
echo -ne '\xdd\x01\x0a\x24\x33\x66\x61\x65\x65\x30\x36\x34\x2d\x63\x34\x66\x37\x2d\x34\x64\x33\x34\x2d\x62\x36\x66\x33\x2d\x38\x64\x38\x31\x63\x32\x62\x36\x61\x31\x35\x64\x12\x04\x4e\x69\x63\x6b\x1a\x0a\x4b\x6f\x6c\x65\x73\x6e\x69\x6b\x6f\x76\x20\x01\x28\xda\x52\x32\x03\x62\x6d\x70\x3a\x0c\x34\x31\x32\x2d\x36\x38\x37\x2d\x35\x30\x30\x37\x40\x01\x4d\x2f\x27\xf2\x5b\x50\x14\x58\x09\x62\x06\x48\x61\x76\x61\x6e\x61\x68\x80\x01\x68\x00\x68\x80\x01\x72\x0a\x50\x69\x74\x74\x73\x62\x75\x72\x67\x68\x7a\x08\x9b\x11\x22\x42\x1f\xe6\x9f\xc2\x81\x01\x28\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x27\x42\x95\x01\x00\x00\x43\x44\x9d\x01\x66\x66\x92\x41\xa0\x01\xce\xdf\xb8\xba\x01\xab\x01\x0d\xcd\xcc\xe2\x41\x0d\xcd\xcc\x4c\x3e\x0d\x00\x00\x80\x3f\x12\x05\x6f\x75\x6e\x63\x65\x12\x05\x63\x61\x72\x61\x74\x12\x04\x67\x72\x61\x6d\xac\x01\xb3\x01\x0b\xa2\x06\x05\x0b\x08\x96\x4a\x0c\x0c\xb4\x01' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'"
|
||||
echo -ne '\x04\x08\x02\x10\x04\x00\x04\x08\x03\x10\x09' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_squares_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'"
|
||||
|
||||
@ -5,3 +5,6 @@ aa0e5a06-cab2-4034-a6a2-48e82b91664e Leonid Kirillov male 1983-06-24 \N +7495027
|
||||
a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502]
|
||||
3faee064-c4f7-4d34-b6f3-8d81c2b6a15d Nick Kolesnikov male 1998-12-26 bmp 412-687-5007\0 1 2018-11-19 05:59:59 20 capricorn ['Havana'] [128,0,128] Pittsburgh [40.517192,-79.949456] 3.1415926535898 50000000000.00 780 18.3 195500007 ['ounce','carat','gram'] [28.35,0.2,1] 9494 []
|
||||
7cfa6856-a54a-4786-b8e5-745159d52278 Isolde Lavrova female 1987-02-09 \N \N 1 \N 32 aquarius [] [255,0,255] [26.000000,-80.000000] 3.1415998935699463 \N 4.5 25.0 -11111111111111 ['Byte','Bit'] [8,1] 702 []
|
||||
0 0
|
||||
2 4
|
||||
3 9
|
||||
|
||||
@ -7,34 +7,37 @@ set -e -o pipefail
|
||||
|
||||
# Run the client.
|
||||
$CLICKHOUSE_CLIENT --multiquery <<'EOF'
|
||||
DROP TABLE IF EXISTS table_00825;
|
||||
DROP TABLE IF EXISTS in_persons_00825;
|
||||
DROP TABLE IF EXISTS in_squares_00825;
|
||||
|
||||
CREATE TABLE table_00825 (uuid UUID,
|
||||
name String,
|
||||
surname String,
|
||||
gender Enum8('male'=1, 'female'=0),
|
||||
birthDate Date,
|
||||
photo Nullable(String),
|
||||
phoneNumber Nullable(FixedString(13)),
|
||||
isOnline UInt8,
|
||||
visitTime Nullable(DateTime),
|
||||
age UInt8,
|
||||
zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823,
|
||||
'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120,
|
||||
'pisces'=219),
|
||||
songs Array(String),
|
||||
color Array(UInt8),
|
||||
hometown LowCardinality(String),
|
||||
location Array(Decimal32(6)),
|
||||
pi Nullable(Float64),
|
||||
lotteryWin Nullable(Decimal64(2)),
|
||||
someRatio Float32,
|
||||
temperature Decimal32(1),
|
||||
randomBigNumber Int64,
|
||||
measureUnits Nested (unit String, coef Float32),
|
||||
nestiness_a_b_c_d Nullable(UInt32),
|
||||
`nestiness_a_B.c_E` Array(UInt32)
|
||||
) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE in_persons_00825 (uuid UUID,
|
||||
name String,
|
||||
surname String,
|
||||
gender Enum8('male'=1, 'female'=0),
|
||||
birthDate Date,
|
||||
photo Nullable(String),
|
||||
phoneNumber Nullable(FixedString(13)),
|
||||
isOnline UInt8,
|
||||
visitTime Nullable(DateTime),
|
||||
age UInt8,
|
||||
zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823,
|
||||
'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120,
|
||||
'pisces'=219),
|
||||
songs Array(String),
|
||||
color Array(UInt8),
|
||||
hometown LowCardinality(String),
|
||||
location Array(Decimal32(6)),
|
||||
pi Nullable(Float64),
|
||||
lotteryWin Nullable(Decimal64(2)),
|
||||
someRatio Float32,
|
||||
temperature Decimal32(1),
|
||||
randomBigNumber Int64,
|
||||
measureUnits Nested (unit String, coef Float32),
|
||||
nestiness_a_b_c_d Nullable(UInt32),
|
||||
`nestiness_a_B.c_E` Array(UInt32)
|
||||
) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
CREATE TABLE in_squares_00825 (number UInt32, square UInt32) ENGINE = MergeTree ORDER BY tuple();
|
||||
EOF
|
||||
|
||||
# To generate the file 00825_protobuf_format_input.insh use the following commands:
|
||||
@ -42,6 +45,8 @@ EOF
|
||||
# build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer
|
||||
source $CURDIR/00825_protobuf_format_input.insh
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM table_00825 ORDER BY uuid;"
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;"
|
||||
$CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;"
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_00825;"
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;"
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;"
|
||||
|
||||
Binary file not shown.
@ -11,46 +11,53 @@ set -e -o pipefail
|
||||
|
||||
# Run the client.
|
||||
$CLICKHOUSE_CLIENT --multiquery <<EOF
|
||||
DROP TABLE IF EXISTS table_00825;
|
||||
DROP TABLE IF EXISTS out_persons_00825;
|
||||
DROP TABLE IF EXISTS out_squares_00825;
|
||||
|
||||
CREATE TABLE table_00825 (uuid UUID,
|
||||
name String,
|
||||
surname String,
|
||||
gender Enum8('male'=1, 'female'=0),
|
||||
birthDate Date,
|
||||
photo Nullable(String),
|
||||
phoneNumber Nullable(FixedString(13)),
|
||||
isOnline UInt8,
|
||||
visitTime Nullable(DateTime),
|
||||
age UInt8,
|
||||
zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823,
|
||||
'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120,
|
||||
'pisces'=219),
|
||||
songs Array(String),
|
||||
color Array(UInt8),
|
||||
hometown LowCardinality(String),
|
||||
location Array(Decimal32(6)),
|
||||
pi Nullable(Float64),
|
||||
lotteryWin Nullable(Decimal64(2)),
|
||||
someRatio Float32,
|
||||
temperature Decimal32(1),
|
||||
randomBigNumber Int64,
|
||||
measureUnits Nested(unit String, coef Float32),
|
||||
nestiness_a_b_c_d Nullable(UInt32),
|
||||
\`nestiness_a_B.c_E\` Array(UInt32)
|
||||
) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE out_persons_00825 (uuid UUID,
|
||||
name String,
|
||||
surname String,
|
||||
gender Enum8('male'=1, 'female'=0),
|
||||
birthDate Date,
|
||||
photo Nullable(String),
|
||||
phoneNumber Nullable(FixedString(13)),
|
||||
isOnline UInt8,
|
||||
visitTime Nullable(DateTime),
|
||||
age UInt8,
|
||||
zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823,
|
||||
'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120,
|
||||
'pisces'=219),
|
||||
songs Array(String),
|
||||
color Array(UInt8),
|
||||
hometown LowCardinality(String),
|
||||
location Array(Decimal32(6)),
|
||||
pi Nullable(Float64),
|
||||
lotteryWin Nullable(Decimal64(2)),
|
||||
someRatio Float32,
|
||||
temperature Decimal32(1),
|
||||
randomBigNumber Int64,
|
||||
measureUnits Nested(unit String, coef Float32),
|
||||
nestiness_a_b_c_d Nullable(UInt32),
|
||||
\`nestiness_a_B.c_E\` Array(UInt32)
|
||||
) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO table_00825 VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000, ['meter', 'centimeter', 'kilometer'], [1, 0.01, 1000], 500, [501, 502]);
|
||||
INSERT INTO table_00825 VALUES (toUUID('c694ad8a-f714-4ea3-907d-fd54fb25d9b5'), 'Natalia', 'Sokolova', 'female', toDate('1992-03-08'), 'jpg', NULL, 0, NULL, 26, 'pisces', [], [100, 200, 50], 'Plymouth', [50.403724, -4.142123], 3.14159, NULL, 0.007, 5.4, -20000000000000, [], [], NULL, []);
|
||||
INSERT INTO table_00825 VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000, ['pound'], [16], 503, []);
|
||||
CREATE TABLE out_squares_00825 (number UInt32, square UInt64) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
SELECT * FROM table_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person';
|
||||
INSERT INTO out_persons_00825 VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000, ['meter', 'centimeter', 'kilometer'], [1, 0.01, 1000], 500, [501, 502]);
|
||||
INSERT INTO out_persons_00825 VALUES (toUUID('c694ad8a-f714-4ea3-907d-fd54fb25d9b5'), 'Natalia', 'Sokolova', 'female', toDate('1992-03-08'), 'jpg', NULL, 0, NULL, 26, 'pisces', [], [100, 200, 50], 'Plymouth', [50.403724, -4.142123], 3.14159, NULL, 0.007, 5.4, -20000000000000, [], [], NULL, []);
|
||||
INSERT INTO out_persons_00825 VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000, ['pound'], [16], 503, []);
|
||||
INSERT INTO out_squares_00825 VALUES (2, 4), (0, 0), (3, 9);
|
||||
|
||||
SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person';
|
||||
SELECT 'ALTERNATIVE->';
|
||||
SELECT * FROM table_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson';
|
||||
SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson';
|
||||
SELECT 'STRINGS->';
|
||||
SELECT * FROM table_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson';
|
||||
SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson';
|
||||
SELECT 'SYNTAX2->';
|
||||
SELECT * FROM table_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person';
|
||||
SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person';
|
||||
SELECT 'SQUARES->';
|
||||
SELECT * FROM out_squares_00825 ORDER BY number FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare';
|
||||
|
||||
DROP TABLE IF EXISTS table_00825;
|
||||
DROP TABLE IF EXISTS out_persons_00825;
|
||||
DROP TABLE IF EXISTS out_squares_00825;
|
||||
EOF
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include "00825_protobuf_format_syntax2.pb.h"
|
||||
|
||||
|
||||
void writeInsertQueryCommand(std::ostream & out, const std::string & format_schema, std::stringstream & delimited_messages)
|
||||
void writeInsertDataQueryForInputTest(std::stringstream & delimited_messages, const std::string & table_name, const std::string & format_schema, std::ostream & out)
|
||||
{
|
||||
out << "echo -ne '";
|
||||
std::string bytes = delimited_messages.str();
|
||||
@ -20,12 +20,12 @@ void writeInsertQueryCommand(std::ostream & out, const std::string & format_sche
|
||||
sprintf(buf, "\\x%02x", static_cast<unsigned char>(c));
|
||||
out << buf;
|
||||
}
|
||||
out << "' | $CLICKHOUSE_CLIENT --query=\"INSERT INTO test.table FORMAT Protobuf"
|
||||
out << "' | $CLICKHOUSE_CLIENT --query=\"INSERT INTO " << table_name << " FORMAT Protobuf"
|
||||
" SETTINGS format_schema = '$CURDIR/"
|
||||
<< format_schema << "'\"" << std::endl;
|
||||
}
|
||||
|
||||
void writeInputInsertQueries(std::ostream & out)
|
||||
void writeInsertDataQueriesForInputTest(std::ostream & out)
|
||||
{
|
||||
std::stringstream ss;
|
||||
{
|
||||
@ -125,7 +125,7 @@ void writeInputInsertQueries(std::ostream & out)
|
||||
google::protobuf::util::SerializeDelimitedToOstream(person, &ss);
|
||||
}
|
||||
|
||||
writeInsertQueryCommand(out, "00825_protobuf_format:Person", ss);
|
||||
writeInsertDataQueryForInputTest(ss, "in_persons_00825", "00825_protobuf_format:Person", out);
|
||||
|
||||
{
|
||||
AltPerson person;
|
||||
@ -189,7 +189,7 @@ void writeInputInsertQueries(std::ostream & out)
|
||||
google::protobuf::util::SerializeDelimitedToOstream(person, &ss);
|
||||
}
|
||||
|
||||
writeInsertQueryCommand(out, "00825_protobuf_format:AltPerson", ss);
|
||||
writeInsertDataQueryForInputTest(ss, "in_persons_00825", "00825_protobuf_format:AltPerson", out);
|
||||
|
||||
{
|
||||
StrPerson person;
|
||||
@ -225,7 +225,7 @@ void writeInputInsertQueries(std::ostream & out)
|
||||
google::protobuf::util::SerializeDelimitedToOstream(person, &ss);
|
||||
}
|
||||
|
||||
writeInsertQueryCommand(out, "00825_protobuf_format:StrPerson", ss);
|
||||
writeInsertDataQueryForInputTest(ss, "in_persons_00825", "00825_protobuf_format:StrPerson", out);
|
||||
|
||||
{
|
||||
Syntax2Person person;
|
||||
@ -262,11 +262,34 @@ void writeInputInsertQueries(std::ostream & out)
|
||||
google::protobuf::util::SerializeDelimitedToOstream(person, &ss);
|
||||
}
|
||||
|
||||
writeInsertQueryCommand(out, "00825_protobuf_format_syntax2:Syntax2Person", ss);
|
||||
writeInsertDataQueryForInputTest(ss, "in_persons_00825", "00825_protobuf_format_syntax2:Syntax2Person", out);
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(2);
|
||||
ns.set_square(4);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &ss);
|
||||
}
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(0);
|
||||
ns.set_square(0);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &ss);
|
||||
}
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(3);
|
||||
ns.set_square(9);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &ss);
|
||||
}
|
||||
|
||||
writeInsertDataQueryForInputTest(ss, "in_squares_00825", "00825_protobuf_format:NumberAndSquare", out);
|
||||
}
|
||||
|
||||
|
||||
void writeOutputReference(std::ostream & out)
|
||||
void writeReferenceForOutputTest(std::ostream & out)
|
||||
{
|
||||
{
|
||||
Person person;
|
||||
@ -637,6 +660,29 @@ void writeOutputReference(std::ostream & out)
|
||||
person.mutable_nestiness()->mutable_a()->mutable_b()->mutable_c()->set_d(503);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(person, &out);
|
||||
}
|
||||
|
||||
out << "SQUARES->" << std::endl;
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(0);
|
||||
ns.set_square(0);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &out);
|
||||
}
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(2);
|
||||
ns.set_square(4);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &out);
|
||||
}
|
||||
|
||||
{
|
||||
NumberAndSquare ns;
|
||||
ns.set_number(3);
|
||||
ns.set_square(9);
|
||||
google::protobuf::util::SerializeDelimitedToOstream(ns, &out);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -676,7 +722,7 @@ int main(int argc, char ** argv)
|
||||
{
|
||||
std::string output_dir;
|
||||
parseCommandLine(argc, argv, output_dir);
|
||||
writeFile(output_dir + "/00825_protobuf_format_input.insh", writeInputInsertQueries);
|
||||
writeFile(output_dir + "/00825_protobuf_format_output.reference", writeOutputReference);
|
||||
writeFile(output_dir + "/00825_protobuf_format_input.insh", writeInsertDataQueriesForInputTest);
|
||||
writeFile(output_dir + "/00825_protobuf_format_output.reference", writeReferenceForOutputTest);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user