thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #53760 from ClibMouse/feature/json-handling-on-big-endian

Browse Source 
Improve JSON-handling on s390x
This commit is contained in:
Robert Schulze 2023-08-28 14:18:45 +02:00 committed by GitHub
commit 53cca12616
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/CMakeLists.txt vendored
View File

@ -136,9 +136,7 @@ add_contrib (aws-cmake
)
add_contrib (base64-cmake base64)
if (NOT ARCH_S390X)
add_contrib (simdjson-cmake simdjson)
endif()
add_contrib (rapidjson-cmake rapidjson)
add_contrib (fastops-cmake fastops)
add_contrib (libuv-cmake libuv)

37
src/DataTypes/Serializations/SerializationDecimalBase.cpp
View File

@ -7,6 +7,7 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <ranges>
namespace DB
{
@ -29,21 +30,13 @@ template <typename T>
void SerializationDecimalBase<T>::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const
{
const typename ColumnType::Container & x = typeid_cast<const ColumnType &>(column).getData();
size_t size = x.size();
if (limit == 0 || offset + limit > size)
if (const size_t size = x.size(); limit == 0 || offset + limit > size)
limit = size - offset;
if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2)
if constexpr (std::endian::native == std::endian::big)
{
for (size_t i = 0; i < limit; i++)
{
auto tmp(x[offset+i]);
char *start = reinterpret_cast<char*>(&tmp);
char *end = start + sizeof(FieldType);
std::reverse(start, end);
ostr.write(reinterpret_cast<const char *>(&tmp), sizeof(FieldType));
}
std::ranges::for_each(
x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & d) { writeBinaryLittleEndian(d, ostr); });
}
else
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(FieldType) * limit);
@ -69,20 +62,14 @@ template <typename T>
void SerializationDecimalBase<T>::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double) const
{
typename ColumnType::Container & x = typeid_cast<ColumnType &>(column).getData();
size_t initial_size = x.size();
const size_t initial_size = x.size();
x.resize(initial_size + limit);
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(FieldType) * limit);
if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2)
{
for (size_t i = 0; i < limit; i++)
{
char *start = reinterpret_cast<char*>(&x[initial_size + i]);
char *end = start + sizeof(FieldType);
std::reverse(start, end);
}
}
const size_t size = istr.readBig(reinterpret_cast<char *>(&x[initial_size]), sizeof(FieldType) * limit);
x.resize(initial_size + size / sizeof(FieldType));
if constexpr (std::endian::native == std::endian::big)
std::ranges::for_each(
x | std::views::drop(initial_size), [](auto & d) { transformEndianness<std::endian::big, std::endian::little>(d); });
}
template class SerializationDecimalBase<Decimal32>;

9
src/DataTypes/Serializations/SerializationNumber.cpp
View File

@ -145,15 +145,8 @@ void SerializationNumber<T>::serializeBinaryBulk(const IColumn & column, WriteBu
if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2)
{
static constexpr auto to_little_endian = [](auto i)
{
transformEndianness<std::endian::little>(i);
return i;
};
std::ranges::for_each(
x | std::views::drop(offset) | std::views::take(limit) | std::views::transform(to_little_endian),
[&ostr](const auto & i) { ostr.write(reinterpret_cast<const char *>(&i), sizeof(typename ColumnVector<T>::ValueType)); });
x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & i) { writeBinaryLittleEndian(i, ostr); });
}
else
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(typename ColumnVector<T>::ValueType) * limit);

25
src/DataTypes/Serializations/SerializationUUID.cpp
View File

@ -7,6 +7,7 @@
#include <IO/WriteHelpers.h>
#include <Common/assert_cast.h>
#include <ranges>
namespace DB
{
@ -136,23 +137,31 @@ void SerializationUUID::deserializeBinary(IColumn & column, ReadBuffer & istr, c
void SerializationUUID::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const
{
const typename ColumnVector<UUID>::Container & x = typeid_cast<const ColumnVector<UUID> &>(column).getData();
size_t size = x.size();
if (limit == 0 || offset + limit > size)
if (const size_t size = x.size(); limit == 0 || offset + limit > size)
limit = size - offset;
if (limit)
if (limit == 0)
return;
if constexpr (std::endian::native == std::endian::big)
{
std::ranges::for_each(
x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & uuid) { writeBinaryLittleEndian(uuid, ostr); });
}
else
ostr.write(reinterpret_cast<const char *>(&x[offset]), sizeof(UUID) * limit);
}
void SerializationUUID::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double /*avg_value_size_hint*/) const
{
typename ColumnVector<UUID>::Container & x = typeid_cast<ColumnVector<UUID> &>(column).getData();
size_t initial_size = x.size();
const size_t initial_size = x.size();
x.resize(initial_size + limit);
size_t size = istr.readBig(reinterpret_cast<char*>(&x[initial_size]), sizeof(UUID) * limit);
const size_t size = istr.readBig(reinterpret_cast<char *>(&x[initial_size]), sizeof(UUID) * limit);
x.resize(initial_size + size / sizeof(UUID));
}
if constexpr (std::endian::native == std::endian::big)
std::ranges::for_each(
x | std::views::drop(initial_size), [](auto & uuid) { transformEndianness<std::endian::big, std::endian::little>(uuid); });
}
}