diff --git a/dbms/include/DB/DataTypes/DataTypeArray.h b/dbms/include/DB/DataTypes/DataTypeArray.h index ecf9a780211..c2a8d55bce2 100644 --- a/dbms/include/DB/DataTypes/DataTypeArray.h +++ b/dbms/include/DB/DataTypes/DataTypeArray.h @@ -61,12 +61,7 @@ public: * Это нужно, так как при реализации вложенных структур, несколько массивов могут иметь общие размеры. */ - /** Записать только значения, без размеров. Вызывающая сторона также должна куда-нибудь записать смещения. */ void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; - - /** Прочитать только значения, без размеров. - * При этом, в column уже заранее должны быть считаны все размеры. - */ void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; /** Записать размеры. */ diff --git a/dbms/include/DB/DataTypes/IDataType.h b/dbms/include/DB/DataTypes/IDataType.h index d5170b5cb01..6f015d3e51c 100644 --- a/dbms/include/DB/DataTypes/IDataType.h +++ b/dbms/include/DB/DataTypes/IDataType.h @@ -60,6 +60,34 @@ public: */ virtual void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const = 0; + + /** More generic methods, suitable for case, when data reside in multiple streams + * (or when it is read/written from/to single stream, but in separate chunks). + * For example, for Array data type, array sizes and array elements are written to two different streams + * (and in case of multidimensional arrays, there are even more streams). + */ + + /** For streams, where data should be read/written, push back to 'out_descriptions' some strings, suitable to be concatenated with file name. + * Example: '.size0', '.size1', ''. + */ + virtual void describeMultipleStreams(std::vector & out_descriptions, size_t level) const + { + out_descriptions.emplace_back(); /// Only one stream. Empty string. + } + + virtual void serializeBinaryBulkWithMupltipleStreams( + IColumn & column, WriteBuffer * streams, size_t num_streams, bool position_independent_encoding, size_t offset, size_t limit) const + { + serializeBinaryBulk(column, streams[0], offset, limit); + } + + virtual void deserializeBinaryBulkWithMultipleStreams( + IColumn & column, ReadBuffer * streams, size_t num_streams, bool position_independent_encoding, size_t limit, double avg_value_size_hint) const + { + deserializeBinaryBulk(column, streams[0], limit, avg_value_size_hint); + } + + /** Serialization/deserialization of individual values. * * These are helper methods for implementation of various formats to input/output for user (like CSV, JSON, etc.). diff --git a/dbms/src/Core/ErrorCodes.cpp b/dbms/src/Core/ErrorCodes.cpp index bc810941ea2..cd8851f14d9 100644 --- a/dbms/src/Core/ErrorCodes.cpp +++ b/dbms/src/Core/ErrorCodes.cpp @@ -353,6 +353,7 @@ namespace ErrorCodes extern const int RESHARDING_NULLABLE_SHARDING_KEY = 348; extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN = 349; extern const int INCOMPATIBLE_SOURCE_TABLES = 350; + extern const int DATA_TYPE_SHOULD_BE_SERIALIZED_INTO_MULTIPLE_STREAMS = 351; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index d9fba8f43cc..276caa76ed1 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -16,6 +16,7 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_READ_ARRAY_FROM_TEXT; + extern const int DATA_TYPE_SHOULD_BE_SERIALIZED_INTO_MULTIPLE_STREAMS; } @@ -97,6 +98,19 @@ void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr) const } +void DataTypeArray::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +{ + throw Exception("Data type Array should be serialized into multiple streams", + ErrorCodes::DATA_TYPE_SHOULD_BE_SERIALIZED_INTO_MULTIPLE_STREAMS); +} + +void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +{ + throw Exception("Data type Array should be serialized into multiple streams", + ErrorCodes::DATA_TYPE_SHOULD_BE_SERIALIZED_INTO_MULTIPLE_STREAMS); +} + + void DataTypeArray::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnArray & column_array = typeid_cast(column);