thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-23 02:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

ClickHouse: IDataType::deserializeBinary now appends to column. Previously it was appending for some types and overwriting for others. [#METR-9091]

Browse Source
This commit is contained in:
Michael Kolupaev 2013-11-26 10:18:09 +00:00
parent 606a2ad942
commit d5ebfa5076
7 changed files with 37 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/include/DB/DataTypes/IDataType.h
View File

@ -50,7 +50,7 @@ public:
*/
virtual void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const = 0;
/** Считать не более limit значений. */
/** Считать не более limit значений и дописать их в конец столбца. */
virtual void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit) const = 0;
/** Текстовая сериализация - для вывода на экран / сохранения в текстовый файл и т. п.

7
dbms/include/DB/DataTypes/IDataTypeNumberFixed.h
View File

@ -54,9 +54,10 @@ public:
void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit) const
{
typename ColumnType::Container_t & x = dynamic_cast<ColumnType &>(column).getData();
x.resize(limit);
size_t size = istr.readBig(reinterpret_cast<char*>(&x[0]), sizeof(typename ColumnType::value_type) * limit);
x.resize(size / sizeof(typename ColumnType::value_type));
size_t initial_size = x.size();
x.resize(initial_size + limit);
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(typename ColumnType::value_type) * limit);
x.resize(initial_size + size / sizeof(typename ColumnType::value_type));
}
ColumnPtr createColumn() const

2
dbms/src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -46,7 +46,7 @@ void DataTypeAggregateFunction::deserializeBinary(IColumn & column, ReadBuffer &
Arena * arena = new Arena;
real_column.set(function);
real_column.addArena(arena);
vec.reserve(limit);
vec.reserve(vec.size() + limit);
size_t size_of_state = function->sizeOfData();

19
dbms/src/DataTypes/DataTypeArray.cpp
View File

@ -74,12 +74,16 @@ void DataTypeArray::deserializeBinary(IColumn & column, ReadBuffer & istr, size_
{
ColumnArray & column_array = dynamic_cast<ColumnArray &>(column);
ColumnArray::Offsets_t & offsets = column_array.getOffsets();
IColumn & nested_column = column_array.getData();
/// Должно быть считано согласнованное с offsets количество значений.
size_t nested_limit = offsets.empty() ? 0 : offsets.back();
nested->deserializeBinary(column_array.getData(), istr, nested_limit);
size_t last_offset = (offsets.empty() ? 0 : offsets.back());
if (last_offset < nested_column.size())
throw Exception("Nested column longer than last offset", ErrorCodes::LOGICAL_ERROR);
size_t nested_limit = last_offset - nested_column.size();
nested->deserializeBinary(nested_column, istr, nested_limit);
if (column_array.getData().size() != nested_limit)
if (column_array.getData().size() != last_offset)
throw Exception("Cannot read all array values", ErrorCodes::CANNOT_READ_ALL_DATA);
}
@ -112,11 +116,12 @@ void DataTypeArray::deserializeOffsets(IColumn & column, ReadBuffer & istr, size
{
ColumnArray & column_array = dynamic_cast<ColumnArray &>(column);
ColumnArray::Offsets_t & offsets = column_array.getOffsets();
offsets.resize(limit);
size_t initial_size = offsets.size();
offsets.resize(initial_size + limit);
size_t i = 0;
ColumnArray::Offset_t current_offset = 0;
while (i < limit && !istr.eof())
size_t i = initial_size;
ColumnArray::Offset_t current_offset = initial_size ? offsets[initial_size - 1] : 0;
while (i < initial_size + limit && !istr.eof())
{
ColumnArray::Offset_t current_size = 0;
readIntBinary(current_size, istr);

7
dbms/src/DataTypes/DataTypeFixedString.cpp
View File

@ -53,15 +53,16 @@ void DataTypeFixedString::deserializeBinary(IColumn & column, ReadBuffer & istr,
{
ColumnFixedString::Chars_t & data = dynamic_cast<ColumnFixedString &>(column).getChars();
size_t initial_size = data.size();
size_t max_bytes = limit * n;
data.resize(max_bytes);
size_t read_bytes = istr.readBig(reinterpret_cast<char *>(&data[0]), max_bytes);
data.resize(initial_size + max_bytes);
size_t read_bytes = istr.readBig(reinterpret_cast<char *>(&data[initial_size]), max_bytes);
if (read_bytes % n != 0)
throw Exception("Cannot read all data of type FixedString",
ErrorCodes::CANNOT_READ_ALL_DATA);
data.resize(read_bytes);
data.resize(initial_size + read_bytes);
}

18
dbms/src/DataTypes/DataTypeNested.cpp
View File

@ -133,13 +133,16 @@ void DataTypeNested::deserializeBinary(IColumn & column, ReadBuffer & istr, size
ColumnNested::Offsets_t & offsets = column_nested.getOffsets();
/// Должно быть считано согласованное с offsets количество значений.
size_t nested_limit = offsets.empty() ? 0 : offsets.back();
size_t last_offset = (offsets.empty() ? 0 : offsets.back());
if (last_offset < column_nested.size())
throw Exception("Nested column longer than last offset", ErrorCodes::LOGICAL_ERROR);
size_t nested_limit = (offsets.empty() ? 0 : offsets.back()) - column_nested.size();
NamesAndTypesList::const_iterator it = nested->begin();
for (size_t i = 0; i < nested->size(); ++i, ++it)
{
it->second->deserializeBinary(*column_nested.getData()[i], istr, nested_limit);
if (column_nested.getData()[i]->size() != nested_limit)
if (column_nested.getData()[i]->size() != last_offset)
throw Exception("Cannot read all nested column values", ErrorCodes::CANNOT_READ_ALL_DATA);
}
}
@ -173,11 +176,12 @@ void DataTypeNested::deserializeOffsets(IColumn & column, ReadBuffer & istr, siz
{
ColumnNested & column_nested = dynamic_cast<ColumnNested &>(column);
ColumnNested::Offsets_t & offsets = column_nested.getOffsets();
offsets.resize(limit);
size_t initial_size = offsets.size();
offsets.resize(initial_size + limit);
size_t i = 0;
ColumnNested::Offset_t current_offset = 0;
while (i < limit && !istr.eof())
size_t i = initial_size;
ColumnNested::Offset_t current_offset = initial_size ? offsets[initial_size - 1] : 0;
while (i < initial_size + limit && !istr.eof())
{
ColumnNested::Offset_t current_size = 0;
readIntBinary(current_size, istr);

9
dbms/src/DataTypes/DataTypeString.cpp
View File

@ -76,10 +76,10 @@ void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr, size
ColumnString::Chars_t & data = column_string.getChars();
ColumnString::Offsets_t & offsets = column_string.getOffsets();
data.reserve(limit * DBMS_APPROX_STRING_SIZE);
offsets.reserve(limit);
data.reserve(data.size() + limit * DBMS_APPROX_STRING_SIZE);
offsets.reserve(offsets.size() + limit);
size_t offset = 0;
size_t offset = data.size();
for (size_t i = 0; i < limit; ++i)
{
if (istr.eof())
@ -91,8 +91,7 @@ void DataTypeString::deserializeBinary(IColumn & column, ReadBuffer & istr, size
offset += size + 1;
offsets.push_back(offset);
if (data.size() < offset)
data.resize(offset);
data.resize(offset);
istr.readStrict(reinterpret_cast<char*>(&data[offset - size - 1]), sizeof(ColumnUInt8::value_type) * size);
data[offset - 1] = 0;