thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed due to review

Browse Source
This commit is contained in:
yariks5s 2023-11-14 13:15:14 +00:00
parent 6f6454e696
commit 22a9f71393
5 changed files with 29 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/Processors/Formats/Impl/NpyRowInputFormat.cpp
View File

@ -32,6 +32,8 @@ namespace
float convertFloat16ToFloat32(uint16_t float16_value)
{
if (float16_value == 0000000000000000)
return float(0);
uint16_t sign = (float16_value >> 15) & 0x1;
uint16_t exponent = (float16_value >> 10) & 0x1F;
uint16_t fraction = float16_value & 0x3FF;
@ -298,26 +300,25 @@ NpyRowInputFormat::NpyRowInputFormat(ReadBuffer & in_, Block header_, Params par
}
template <typename ColumnValue, typename DataValue>
void NpyRowInputFormat::readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness, bool isFloat16)
void NpyRowInputFormat::readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness)
{
if (!isFloat16)
{
DataValue value;
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue((static_cast<ColumnValue>(value)));
}
DataValue value;
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
{
uint16_t value;
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue(static_cast<ColumnValue>(convertFloat16ToFloat32(value)));
}
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue((static_cast<ColumnValue>(value)));
}
template <typename ColumnValue>
void NpyRowInputFormat::readBinaryValueAndInsertFloat16(MutableColumnPtr column, NumpyDataType::Endianness endianness)
{
uint16_t value;
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue(static_cast<ColumnValue>(convertFloat16ToFloat32(value)));
}
template <typename T>
@ -344,7 +345,7 @@ void NpyRowInputFormat::readAndInsertFloat(IColumn * column, const DataTypePtr &
{
switch (npy_type.getTypeIndex())
{
case NumpyDataTypeIndex::Float16: readBinaryValueAndInsert<T, Float32>(column->getPtr(), npy_type.getEndianness(), true); break;
case NumpyDataTypeIndex::Float16: readBinaryValueAndInsertFloat16<T>(column->getPtr(), npy_type.getEndianness()); break;
case NumpyDataTypeIndex::Float32: readBinaryValueAndInsert<T, Float32>(column->getPtr(), npy_type.getEndianness()); break;
case NumpyDataTypeIndex::Float64: readBinaryValueAndInsert<T, Float64>(column->getPtr(), npy_type.getEndianness()); break;
default:

5
src/Processors/Formats/Impl/NpyRowInputFormat.h
View File

@ -43,7 +43,10 @@ private:
void readAndInsertString(MutableColumnPtr column, const DataTypePtr & data_type, const NumpyDataType & npy_type, bool is_fixed);
template <typename ColumnValue, typename DataValue>
void readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness, bool isFloat16 = false);
void readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness);
template <typename ColumnValue>
void readBinaryValueAndInsertFloat16(MutableColumnPtr column, NumpyDataType::Endianness endianness);
void readRows(MutableColumns & columns);

3
tests/queries/0_stateless/02895_npy_format.reference
View File

@ -86,3 +86,6 @@ c
1
[2.199219,1.099609,3.300781]
[4.25,3.34961,6.628906]
inf
nan
0

2
tests/queries/0_stateless/02895_npy_format.sh
View File

@ -58,3 +58,5 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy',
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS"
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/float_16.npy')"
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/npy_inf_nan_null.npy')"

BIN
tests/queries/0_stateless/data_npy/npy_inf_nan_null.npy Normal file
View File

Binary file not shown.