diff --git a/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h b/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h index b081f964226..ef04edc0d57 100644 --- a/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h @@ -46,8 +46,8 @@ public: void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeBinary(IColumn & column, ReadBuffer & istr) const override; - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override; diff --git a/dbms/include/DB/DataTypes/DataTypeArray.h b/dbms/include/DB/DataTypes/DataTypeArray.h index fb16c2d0ce1..ecf9a780211 100644 --- a/dbms/include/DB/DataTypes/DataTypeArray.h +++ b/dbms/include/DB/DataTypes/DataTypeArray.h @@ -62,15 +62,15 @@ public: */ /** Записать только значения, без размеров. Вызывающая сторона также должна куда-нибудь записать смещения. */ - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; /** Прочитать только значения, без размеров. * При этом, в column уже заранее должны быть считаны все размеры. */ - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; /** Записать размеры. */ - void serializeOffsets(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const; + void serializeOffsets(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const; /** Прочитать размеры. Вызывайте этот метод перед чтением значений. */ void deserializeOffsets(IColumn & column, ReadBuffer & istr, size_t limit) const; diff --git a/dbms/include/DB/DataTypes/DataTypeEnum.h b/dbms/include/DB/DataTypes/DataTypeEnum.h index a84255c3557..da0109ec607 100644 --- a/dbms/include/DB/DataTypes/DataTypeEnum.h +++ b/dbms/include/DB/DataTypes/DataTypeEnum.h @@ -99,8 +99,8 @@ public: void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override; - void serializeBinary(const IColumn & column, WriteBuffer & ostr, const size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, const size_t limit, const double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, const size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, const size_t limit, const double avg_value_size_hint) const override; size_t getSizeOfField() const override { return sizeof(FieldType); } diff --git a/dbms/include/DB/DataTypes/DataTypeFixedString.h b/dbms/include/DB/DataTypes/DataTypeFixedString.h index 3fd85e5ae06..61cb59a9883 100644 --- a/dbms/include/DB/DataTypes/DataTypeFixedString.h +++ b/dbms/include/DB/DataTypes/DataTypeFixedString.h @@ -41,8 +41,8 @@ public: void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeBinary(IColumn & column, ReadBuffer & istr) const override; - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/include/DB/DataTypes/DataTypeNull.h b/dbms/include/DB/DataTypes/DataTypeNull.h index 3046f5f1f2a..380b4355491 100644 --- a/dbms/include/DB/DataTypes/DataTypeNull.h +++ b/dbms/include/DB/DataTypes/DataTypeNull.h @@ -36,8 +36,8 @@ public: return std::make_shared(); } - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; ColumnPtr createColumn() const override { diff --git a/dbms/include/DB/DataTypes/DataTypeNullable.h b/dbms/include/DB/DataTypes/DataTypeNullable.h index 841c8e41f8b..8c4a63f4a73 100644 --- a/dbms/include/DB/DataTypes/DataTypeNullable.h +++ b/dbms/include/DB/DataTypes/DataTypeNullable.h @@ -22,8 +22,8 @@ public: DataTypePtr clone() const override { return std::make_shared(nested_data_type->clone()); } /// Bulk serialization and deserialization is processing only nested columns. You should process null byte map separately. - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; void serializeBinary(const Field & field, WriteBuffer & ostr) const override { nested_data_type->serializeBinary(field, ostr); } void deserializeBinary(Field & field, ReadBuffer & istr) const override { nested_data_type->deserializeBinary(field, istr); } diff --git a/dbms/include/DB/DataTypes/DataTypeString.h b/dbms/include/DB/DataTypes/DataTypeString.h index da855444d11..cf3e38b4c72 100644 --- a/dbms/include/DB/DataTypes/DataTypeString.h +++ b/dbms/include/DB/DataTypes/DataTypeString.h @@ -28,8 +28,8 @@ public: void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeBinary(IColumn & column, ReadBuffer & istr) const override; - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/include/DB/DataTypes/DataTypeTuple.h b/dbms/include/DB/DataTypes/DataTypeTuple.h index d26f684490f..05c0fa60214 100644 --- a/dbms/include/DB/DataTypes/DataTypeTuple.h +++ b/dbms/include/DB/DataTypes/DataTypeTuple.h @@ -39,13 +39,13 @@ public: void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override; - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; /** limit обязательно должен быть в точности равен количеству сериализованных значений. * Именно из-за этого (невозможности читать меньший кусок записанных данных), Tuple не могут быть использованы для хранения данных в таблицах. * (Хотя могут быть использованы для передачи данных по сети в Native формате.) */ - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; ColumnPtr createColumn() const override; ColumnPtr createConstColumn(size_t size, const Field & field) const override; diff --git a/dbms/include/DB/DataTypes/IDataType.h b/dbms/include/DB/DataTypes/IDataType.h index 8ccc3ce76a6..dd678ab0f83 100644 --- a/dbms/include/DB/DataTypes/IDataType.h +++ b/dbms/include/DB/DataTypes/IDataType.h @@ -53,14 +53,18 @@ public: * offset + limit может быть больше размера столбца * - в этом случае, столбец сериализуется до конца. */ - virtual void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const = 0; + virtual void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const = 0; /** Считать не более limit значений и дописать их в конец столбца. * avg_value_size_hint - если не 0, то может использоваться, чтобы избежать реаллокаций при чтении строкового столбца. */ - virtual void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const = 0; + virtual void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const = 0; - /// Сериализация единичных значений. + /** Serialization/deserialization of individual values. + * For complex data types (like arrays) it may differ from bulk serde. + * For example, if you serialize single array, it will be represented as its size and values in single contiguous stream, + * but if you serialize column with arrays as bulk, then sizes and values will be written to separate streams. + */ /// Для бинарной сериализации есть два варианта. Один вариант работает с Field. virtual void serializeBinary(const Field & field, WriteBuffer & ostr) const = 0; diff --git a/dbms/include/DB/DataTypes/IDataTypeDummy.h b/dbms/include/DB/DataTypes/IDataTypeDummy.h index 1cbae93974e..00299ebcdf5 100644 --- a/dbms/include/DB/DataTypes/IDataTypeDummy.h +++ b/dbms/include/DB/DataTypes/IDataTypeDummy.h @@ -25,10 +25,10 @@ public: void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override { throwNoSerialization(); } void deserializeBinary(IColumn & column, ReadBuffer & istr) const override { throwNoSerialization(); } - void serializeBinary(const IColumn & column, WriteBuffer & ostr, - size_t offset = 0, size_t limit = 0) const override { throwNoSerialization(); } + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, + size_t offset, size_t limit) const override { throwNoSerialization(); } - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override { throwNoSerialization(); } + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override { throwNoSerialization(); } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override { throwNoSerialization(); } diff --git a/dbms/include/DB/DataTypes/IDataTypeNumberFixed.h b/dbms/include/DB/DataTypes/IDataTypeNumberFixed.h index e93b2d9f12e..84a06a15995 100644 --- a/dbms/include/DB/DataTypes/IDataTypeNumberFixed.h +++ b/dbms/include/DB/DataTypes/IDataTypeNumberFixed.h @@ -57,7 +57,7 @@ public: static_cast(column).getData().push_back(x); } - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override { const typename ColumnType::Container_t & x = typeid_cast(column).getData(); @@ -69,7 +69,7 @@ public: ostr.write(reinterpret_cast(&x[offset]), sizeof(typename ColumnType::value_type) * limit); } - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override { typename ColumnType::Container_t & x = typeid_cast(column).getData(); size_t initial_size = x.size(); @@ -114,12 +114,12 @@ public: throw Exception{"Method deserializeBinary is not supported for DataTypeVoid", ErrorCodes::NOT_IMPLEMENTED}; } - void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override { throw Exception{"Method serializeBinary is not supported for DataTypeVoid", ErrorCodes::NOT_IMPLEMENTED}; } - void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override { throw Exception{"Method deserializeBinary is not supported for DataTypeVoid", ErrorCodes::NOT_IMPLEMENTED}; } diff --git a/dbms/src/DataStreams/NativeBlockInputStream.cpp b/dbms/src/DataStreams/NativeBlockInputStream.cpp index ca65fa9d7ef..644bbde8980 100644 --- a/dbms/src/DataStreams/NativeBlockInputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockInputStream.cpp @@ -55,7 +55,7 @@ void NativeBlockInputStream::readData(const IDataType & type, IColumn & column, IColumn & nested_col = *nullable_col.getNestedColumn(); IColumn & null_map = *nullable_col.getNullMapColumn(); - DataTypeUInt8{}.deserializeBinary(null_map, istr, rows, 0); + DataTypeUInt8{}.deserializeBinaryBulk(null_map, istr, rows, 0); readData(nested_type, nested_col, istr, rows); @@ -66,7 +66,7 @@ void NativeBlockInputStream::readData(const IDataType & type, IColumn & column, /** Для массивов требуется сначала десериализовать смещения, а потом значения. */ IColumn & offsets_column = *typeid_cast(column).getOffsetsColumn(); - type_arr->getOffsetsType()->deserializeBinary(offsets_column, istr, rows, 0); + type_arr->getOffsetsType()->deserializeBinaryBulk(offsets_column, istr, rows, 0); if (offsets_column.size() != rows) throw Exception("Cannot read all data in NativeBlockInputStream.", ErrorCodes::CANNOT_READ_ALL_DATA); @@ -79,7 +79,7 @@ void NativeBlockInputStream::readData(const IDataType & type, IColumn & column, typeid_cast(column).getOffsets()[rows - 1]); } else - type.deserializeBinary(column, istr, rows, 0); /// TODO Использовать avg_value_size_hint. + type.deserializeBinaryBulk(column, istr, rows, 0); /// TODO Использовать avg_value_size_hint. if (column.size() != rows) throw Exception("Cannot read all data in NativeBlockInputStream.", ErrorCodes::CANNOT_READ_ALL_DATA); diff --git a/dbms/src/DataStreams/NativeBlockOutputStream.cpp b/dbms/src/DataStreams/NativeBlockOutputStream.cpp index 0b3ef5f01a7..bc3b60859d6 100644 --- a/dbms/src/DataStreams/NativeBlockOutputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockOutputStream.cpp @@ -62,7 +62,7 @@ void NativeBlockOutputStream::writeData(const IDataType & type, const ColumnPtr const ColumnPtr & nested_col = nullable_col.getNestedColumn(); const IColumn & null_map = *nullable_col.getNullMapColumn(); - DataTypeUInt8{}.serializeBinary(null_map, ostr, offset, limit); + DataTypeUInt8{}.serializeBinaryBulk(null_map, ostr, offset, limit); writeData(nested_type, nested_col, ostr, offset, limit); } @@ -71,7 +71,7 @@ void NativeBlockOutputStream::writeData(const IDataType & type, const ColumnPtr /** Для массивов требуется сначала сериализовать смещения, а потом значения. */ const ColumnArray & column_array = typeid_cast(*full_column); - type_arr->getOffsetsType()->serializeBinary(*column_array.getOffsetsColumn(), ostr, offset, limit); + type_arr->getOffsetsType()->serializeBinaryBulk(*column_array.getOffsetsColumn(), ostr, offset, limit); if (!typeid_cast(*full_column).getData().empty()) { @@ -111,7 +111,7 @@ void NativeBlockOutputStream::writeData(const IDataType & type, const ColumnPtr } } else - type.serializeBinary(*full_column, ostr, offset, limit); + type.serializeBinaryBulk(*full_column, ostr, offset, limit); } diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index 48214fb33db..901cbfe3f34 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -80,7 +80,7 @@ void DataTypeAggregateFunction::deserializeBinary(IColumn & column, ReadBuffer & column_concrete.getData().push_back(place); } -void DataTypeAggregateFunction::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeAggregateFunction::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnAggregateFunction & real_column = typeid_cast(column); const ColumnAggregateFunction::Container_t & vec = real_column.getData(); @@ -95,7 +95,7 @@ void DataTypeAggregateFunction::serializeBinary(const IColumn & column, WriteBuf function->serialize(*it, ostr); } -void DataTypeAggregateFunction::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeAggregateFunction::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnAggregateFunction & real_column = typeid_cast(column); ColumnAggregateFunction::Container_t & vec = real_column.getData(); diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index 3c7f6578eee..d9fba8f43cc 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -97,7 +97,7 @@ void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr) const } -void DataTypeArray::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeArray::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnArray & column_array = typeid_cast(column); const ColumnArray::Offsets_t & offsets = column_array.getOffsets(); @@ -121,11 +121,11 @@ void DataTypeArray::serializeBinary(const IColumn & column, WriteBuffer & ostr, : 0; if (limit == 0 || nested_limit) - nested->serializeBinary(column_array.getData(), ostr, nested_offset, nested_limit); + nested->serializeBinaryBulk(column_array.getData(), ostr, nested_offset, nested_limit); } -void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeArray::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnArray & column_array = typeid_cast(column); ColumnArray::Offsets_t & offsets = column_array.getOffsets(); @@ -136,7 +136,7 @@ void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr, size_ if (last_offset < nested_column.size()) throw Exception("Nested column longer than last offset", ErrorCodes::LOGICAL_ERROR); size_t nested_limit = last_offset - nested_column.size(); - nested->deserializeBinary(nested_column, istr, nested_limit, 0); + nested->deserializeBinaryBulk(nested_column, istr, nested_limit, 0); if (column_array.getData().size() != last_offset) throw Exception("Cannot read all array values", ErrorCodes::CANNOT_READ_ALL_DATA); diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index f73435d75e2..4c9662490c8 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -205,7 +205,7 @@ void DataTypeEnum::deserializeTextCSV(IColumn & column, ReadBuffer & istr, } template -void DataTypeEnum::serializeBinary( +void DataTypeEnum::serializeBinaryBulk( const IColumn & column, WriteBuffer & ostr, const size_t offset, size_t limit) const { const auto & x = typeid_cast(column).getData(); @@ -218,7 +218,7 @@ void DataTypeEnum::serializeBinary( } template -void DataTypeEnum::deserializeBinary( +void DataTypeEnum::deserializeBinaryBulk( IColumn & column, ReadBuffer & istr, const size_t limit, const double avg_value_size_hint) const { auto & x = typeid_cast(column).getData(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index f0b67d36aae..0a5cce5e43a 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -69,7 +69,7 @@ void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr) } -void DataTypeFixedString::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeFixedString::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnFixedString::Chars_t & data = typeid_cast(column).getChars(); @@ -82,7 +82,7 @@ void DataTypeFixedString::serializeBinary(const IColumn & column, WriteBuffer & } -void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeFixedString::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnFixedString::Chars_t & data = typeid_cast(column).getChars(); diff --git a/dbms/src/DataTypes/DataTypeNull.cpp b/dbms/src/DataTypes/DataTypeNull.cpp index 994bd1e97c5..f1353368124 100644 --- a/dbms/src/DataTypes/DataTypeNull.cpp +++ b/dbms/src/DataTypes/DataTypeNull.cpp @@ -3,7 +3,7 @@ namespace DB { -void DataTypeNull::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeNull::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { size_t size = column.size(); @@ -15,7 +15,7 @@ void DataTypeNull::serializeBinary(const IColumn & column, WriteBuffer & ostr, s writeBinary(x, ostr); } -void DataTypeNull::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeNull::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnNull & null_col = static_cast(column); diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 0200bde1eb6..cfdf7066a48 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -16,16 +16,16 @@ DataTypeNullable::DataTypeNullable(DataTypePtr nested_data_type_) { } -void DataTypeNullable::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeNullable::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnNullable & col = static_cast(column); - nested_data_type->serializeBinary(*col.getNestedColumn(), ostr, offset, limit); + nested_data_type->serializeBinaryBulk(*col.getNestedColumn(), ostr, offset, limit); } -void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeNullable::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnNullable & col = static_cast(column); - nested_data_type->deserializeBinary(*col.getNestedColumn(), istr, limit, avg_value_size_hint); + nested_data_type->deserializeBinaryBulk(*col.getNestedColumn(), istr, limit, avg_value_size_hint); } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 4a80971c288..9b6f166425a 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -74,7 +74,7 @@ void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr) cons } -void DataTypeString::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeString::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnString & column_string = typeid_cast(column); const ColumnString::Chars_t & data = column_string.getChars(); @@ -167,7 +167,7 @@ static NO_INLINE void deserializeBinarySSE2(ColumnString::Chars_t & data, Column } -void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeString::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnString & column_string = typeid_cast(column); ColumnString::Chars_t & data = column_string.getChars(); diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 6e08c136aa7..4472f7c45b3 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -216,14 +216,14 @@ void DataTypeTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, cons }); } -void DataTypeTuple::serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +void DataTypeTuple::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const ColumnTuple & real_column = static_cast(column); for (size_t i = 0, size = elems.size(); i < size; ++i) NativeBlockOutputStream::writeData(*elems[i], real_column.getData().safeGetByPosition(i).column, ostr, offset, limit); } -void DataTypeTuple::deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const +void DataTypeTuple::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const { ColumnTuple & real_column = static_cast(column); for (size_t i = 0, size = elems.size(); i < size; ++i) diff --git a/dbms/src/DataTypes/tests/data_type_string.cpp b/dbms/src/DataTypes/tests/data_type_string.cpp index 1e7ee4fcdb4..e852b3e2968 100644 --- a/dbms/src/DataTypes/tests/data_type_string.cpp +++ b/dbms/src/DataTypes/tests/data_type_string.cpp @@ -39,7 +39,7 @@ try WriteBufferFromOStream out_buf(ostr); stopwatch.restart(); - data_type.serializeBinary(*column, out_buf); + data_type.serializeBinaryBulk(*column, out_buf); stopwatch.stop(); std::cout << "Writing, elapsed: " << stopwatch.elapsedSeconds() << std::endl; @@ -52,7 +52,7 @@ try ReadBufferFromIStream in_buf(istr); stopwatch.restart(); - data_type.deserializeBinary(*column, in_buf, n, 0); + data_type.deserializeBinaryBulk(*column, in_buf, n, 0); stopwatch.stop(); std::cout << "Reading, elapsed: " << stopwatch.elapsedSeconds() << std::endl; diff --git a/dbms/src/DataTypes/tests/data_types_number_fixed.cpp b/dbms/src/DataTypes/tests/data_types_number_fixed.cpp index 956ecae72b1..e7e7d6132e9 100644 --- a/dbms/src/DataTypes/tests/data_types_number_fixed.cpp +++ b/dbms/src/DataTypes/tests/data_types_number_fixed.cpp @@ -27,7 +27,7 @@ int main(int argc, char ** argv) WriteBufferFromOStream out_buf(ostr); stopwatch.restart(); - data_type.serializeBinary(*column, out_buf); + data_type.serializeBinaryBulk(*column, out_buf); stopwatch.stop(); std::cout << "Elapsed: " << stopwatch.elapsedSeconds() << std::endl; diff --git a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp index 2c262c034c5..a0bb93d3bc4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp @@ -72,7 +72,7 @@ public: size_t read(size_t rows) { ColumnPtr column = type->createColumn(); - type->deserializeBinary(*column, uncompressed_hashing_buf, rows, 0); + type->deserializeBinaryBulk(*column, uncompressed_hashing_buf, rows, 0); return column->size(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 0c0c961b325..2b7f7dc2370 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -417,7 +417,7 @@ void MergeTreeReader::readData(const String & name, const IDataType & type, ICol Stream & stream = *(streams.at(filename)); stream.seekToMark(from_mark); IColumn & col8 = *(nullable_col.getNullMapColumn()); - DataTypeUInt8{}.deserializeBinary(col8, *stream.data_buffer, max_rows_to_read, 0); + DataTypeUInt8{}.deserializeBinaryBulk(col8, *stream.data_buffer, max_rows_to_read, 0); /// Then read data. readData(name, nested_type, nested_col, from_mark, max_rows_to_read, level, read_offsets); @@ -483,7 +483,7 @@ void MergeTreeReader::readData(const String & name, const IDataType & type, ICol double & avg_value_size_hint = avg_value_size_hints[name]; stream.seekToMark(from_mark); - type.deserializeBinary(column, *stream.data_buffer, max_rows_to_read, avg_value_size_hint); + type.deserializeBinaryBulk(column, *stream.data_buffer, max_rows_to_read, avg_value_size_hint); /// Вычисление подсказки о среднем размере значения. size_t column_size = column.size(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 79dbfbccf7d..52b121eb158 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -170,7 +170,7 @@ void IMergedBlockOutputStream::writeDataImpl( writeIntBinary(stream.compressed.offset(), stream.marks); } - DataTypeUInt8{}.serializeBinary(*(nullable_col.getNullMapColumn()), stream.compressed); + DataTypeUInt8{}.serializeBinaryBulk(*(nullable_col.getNullMapColumn()), stream.compressed); /// Чтобы вместо засечек, указывающих на конец сжатого блока, были засечки, указывающие на начало следующего. stream.compressed.nextIfAtEnd(); @@ -253,7 +253,7 @@ void IMergedBlockOutputStream::writeDataImpl( writeIntBinary(stream.compressed.offset(), stream.marks); } - type.serializeBinary(column, stream.compressed, prev_mark, limit); + type.serializeBinaryBulk(column, stream.compressed, prev_mark, limit); /// Чтобы вместо засечек, указывающих на конец сжатого блока, были засечки, указывающие на начало следующего. stream.compressed.nextIfAtEnd(); diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 4bae9bc023b..abd7f88cdab 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -359,7 +359,7 @@ void LogBlockInputStream::readData(const String & name, const IDataType & type, ColumnNullable & nullable_col = static_cast(column); IColumn & nested_col = *nullable_col.getNestedColumn(); - DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullMapColumn(), + DataTypeUInt8{}.deserializeBinaryBulk(*nullable_col.getNullMapColumn(), streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed, max_rows_to_read, 0); /// Then read data. readData(name, nested_type, nested_col, max_rows_to_read, level, read_offsets); @@ -384,7 +384,7 @@ void LogBlockInputStream::readData(const String & name, const IDataType & type, level + 1); } else - type.deserializeBinary(column, streams[name]->compressed, max_rows_to_read, 0); /// TODO Использовать avg_value_size_hint. + type.deserializeBinaryBulk(column, streams[name]->compressed, max_rows_to_read, 0); /// TODO Использовать avg_value_size_hint. } @@ -490,8 +490,7 @@ void LogBlockOutputStream::writeData(const String & name, const IDataType & type out_null_marks.emplace_back(storage.files[filename].column_index, mark); - DataTypeUInt8{}.serializeBinary(*nullable_col.getNullMapColumn(), - streams[filename]->compressed); + DataTypeUInt8{}.serializeBinaryBulk(*nullable_col.getNullMapColumn(), streams[filename]->compressed); streams[filename]->compressed.next(); /// Then write data. @@ -527,7 +526,7 @@ void LogBlockOutputStream::writeData(const String & name, const IDataType & type out_marks.push_back(std::make_pair(storage.files[name].column_index, mark)); - type.serializeBinary(column, streams[name]->compressed); + type.serializeBinaryBulk(column, streams[name]->compressed); streams[name]->compressed.next(); } } diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index b7385d33355..8501a83074d 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -300,7 +300,7 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty IColumn & nested_col = *nullable_col.getNestedColumn(); /// First read from the null map. - DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullMapColumn(), + DataTypeUInt8{}.deserializeBinaryBulk(*nullable_col.getNullMapColumn(), streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed, limit, 0); /// Then read data. @@ -328,7 +328,7 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty } } else - type.deserializeBinary(column, streams[name]->compressed, limit, 0); /// TODO Использовать avg_value_size_hint. + type.deserializeBinaryBulk(column, streams[name]->compressed, limit, 0); /// TODO Использовать avg_value_size_hint. } @@ -372,7 +372,7 @@ void TinyLogBlockOutputStream::writeData(const String & name, const IDataType & const ColumnNullable & nullable_col = static_cast(column); const IColumn & nested_col = *nullable_col.getNestedColumn(); - DataTypeUInt8{}.serializeBinary(*nullable_col.getNullMapColumn(), + DataTypeUInt8{}.serializeBinaryBulk(*nullable_col.getNullMapColumn(), streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed); /// Then write data. @@ -394,7 +394,7 @@ void TinyLogBlockOutputStream::writeData(const String & name, const IDataType & writeData(name, *type_arr->getNestedType(), typeid_cast(column).getData(), offset_columns, level + 1); } else - type.serializeBinary(column, streams[name]->compressed); + type.serializeBinaryBulk(column, streams[name]->compressed); }