thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-11 17:02:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement big-endian support for transform

Browse Source
This commit is contained in:
ltrk2 2023-08-03 12:38:43 -07:00
parent 968c79cf3f
commit 6dc74fb5cc
3 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/DataTypes/Serializations/SerializationArray.cpp
View File

@ -129,7 +129,7 @@ namespace
for (size_t i = offset; i < end; ++i)
{
ColumnArray::Offset current_offset = offset_values[i];
writeIntBinary(current_offset - prev_offset, ostr);
writeBinaryLittleEndian(current_offset - prev_offset, ostr);
prev_offset = current_offset;
}
}
@ -145,7 +145,7 @@ namespace
while (i < initial_size + limit && !istr.eof())
{
ColumnArray::Offset current_size = 0;
readIntBinary(current_size, istr);
readBinaryLittleEndian(current_size, istr);
if (unlikely(current_size > MAX_ARRAY_SIZE))
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size is too large: {}", current_size);

8
src/DataTypes/Serializations/SerializationNumber.cpp
View File

@ -106,28 +106,28 @@ void SerializationNumber<T>::serializeBinary(const Field & field, WriteBuffer &
{
/// ColumnVector<T>::ValueType is a narrower type. For example, UInt8, when the Field type is UInt64
typename ColumnVector<T>::ValueType x = static_cast<typename ColumnVector<T>::ValueType>(field.get<FieldType>());
writeBinary(x, ostr);
writeBinaryLittleEndian(x, ostr);
}
template <typename T>
void SerializationNumber<T>::deserializeBinary(Field & field, ReadBuffer & istr, const FormatSettings &) const
{
typename ColumnVector<T>::ValueType x;
readBinary(x, istr);
readBinaryLittleEndian(x, istr);
field = NearestFieldType<FieldType>(x);
}
template <typename T>
void SerializationNumber<T>::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
{
writeBinary(assert_cast<const ColumnVector<T> &>(column).getData()[row_num], ostr);
writeBinaryLittleEndian(assert_cast<const ColumnVector<T> &>(column).getData()[row_num], ostr);
}
template <typename T>
void SerializationNumber<T>::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
{
typename ColumnVector<T>::ValueType x;
readBinary(x, istr);
readBinaryLittleEndian(x, istr);
assert_cast<ColumnVector<T> &>(column).getData().push_back(x);
}

14
src/Functions/transform.cpp
View File

@ -764,9 +764,8 @@ namespace
}
/// Note: Doesn't check the duplicates in the `from` array.
WhichDataType which(from_type);
if (isNativeNumber(which) || which.isDecimal32() || which.isDecimal64())
/// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64
if (WhichDataType which(from_type); isNativeNumber(which) || which.isDecimal32() || which.isDecimal64())
{
cache.table_num_to_idx = std::make_unique<Cache::NumToIdx>();
auto & table = *cache.table_num_to_idx;
@ -774,10 +773,13 @@ namespace
{
if (applyVisitor(FieldVisitorAccurateEquals(), (*cache.from_column)[i], (*from_column_uncasted)[i]))
{
/// Field may be of Float type, but for the purpose of bitwise equality we can treat them as UInt64
StringRef ref = cache.from_column->getDataAt(i);
UInt64 key = 0;
memcpy(&key, ref.data, ref.size);
auto * dst = reinterpret_cast<char *>(&key);
const auto ref = cache.from_column->getDataAt(i);
if constexpr (std::endian::native == std::endian::big)
dst += sizeof(key) - ref.size;
memcpy(dst, ref.data, ref.size);
table[key] = i;
}
}