thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed due to review

Browse Source
This commit is contained in:
yariks5s 2023-11-14 13:15:14 +00:00
parent 6f6454e696
commit 22a9f71393
5 changed files with 29 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/Processors/Formats/Impl/NpyRowInputFormat.cpp
View File

@ -32,6 +32,8 @@ namespace
float convertFloat16ToFloat32(uint16_t float16_value) float convertFloat16ToFloat32(uint16_t float16_value)
{ {
if (float16_value == 0000000000000000)
return float(0);
uint16_t sign = (float16_value >> 15) & 0x1; uint16_t sign = (float16_value >> 15) & 0x1;
uint16_t exponent = (float16_value >> 10) & 0x1F; uint16_t exponent = (float16_value >> 10) & 0x1F;
uint16_t fraction = float16_value & 0x3FF; uint16_t fraction = float16_value & 0x3FF;
@ -298,26 +300,25 @@ NpyRowInputFormat::NpyRowInputFormat(ReadBuffer & in_, Block header_, Params par
} }
template <typename ColumnValue, typename DataValue> template <typename ColumnValue, typename DataValue>
void NpyRowInputFormat::readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness, bool isFloat16) void NpyRowInputFormat::readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness)
{ {
if (!isFloat16) DataValue value;
{ if (endianness == NumpyDataType::Endianness::BIG)
DataValue value; readBinaryBigEndian(value, *in);
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue((static_cast<ColumnValue>(value)));
}
else else
{ readBinaryLittleEndian(value, *in);
uint16_t value; assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue((static_cast<ColumnValue>(value)));
if (endianness == NumpyDataType::Endianness::BIG) }
readBinaryBigEndian(value, *in);
else template <typename ColumnValue>
readBinaryLittleEndian(value, *in); void NpyRowInputFormat::readBinaryValueAndInsertFloat16(MutableColumnPtr column, NumpyDataType::Endianness endianness)
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue(static_cast<ColumnValue>(convertFloat16ToFloat32(value))); {
} uint16_t value;
if (endianness == NumpyDataType::Endianness::BIG)
readBinaryBigEndian(value, *in);
else
readBinaryLittleEndian(value, *in);
assert_cast<ColumnVector<ColumnValue> &>(*column).insertValue(static_cast<ColumnValue>(convertFloat16ToFloat32(value)));
} }
template <typename T> template <typename T>
@ -344,7 +345,7 @@ void NpyRowInputFormat::readAndInsertFloat(IColumn * column, const DataTypePtr &
{ {
switch (npy_type.getTypeIndex()) switch (npy_type.getTypeIndex())
{ {
case NumpyDataTypeIndex::Float16: readBinaryValueAndInsert<T, Float32>(column->getPtr(), npy_type.getEndianness(), true); break; case NumpyDataTypeIndex::Float16: readBinaryValueAndInsertFloat16<T>(column->getPtr(), npy_type.getEndianness()); break;
case NumpyDataTypeIndex::Float32: readBinaryValueAndInsert<T, Float32>(column->getPtr(), npy_type.getEndianness()); break; case NumpyDataTypeIndex::Float32: readBinaryValueAndInsert<T, Float32>(column->getPtr(), npy_type.getEndianness()); break;
case NumpyDataTypeIndex::Float64: readBinaryValueAndInsert<T, Float64>(column->getPtr(), npy_type.getEndianness()); break; case NumpyDataTypeIndex::Float64: readBinaryValueAndInsert<T, Float64>(column->getPtr(), npy_type.getEndianness()); break;
default: default:

5
src/Processors/Formats/Impl/NpyRowInputFormat.h
View File

@ -43,7 +43,10 @@ private:
void readAndInsertString(MutableColumnPtr column, const DataTypePtr & data_type, const NumpyDataType & npy_type, bool is_fixed); void readAndInsertString(MutableColumnPtr column, const DataTypePtr & data_type, const NumpyDataType & npy_type, bool is_fixed);
template <typename ColumnValue, typename DataValue> template <typename ColumnValue, typename DataValue>
void readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness, bool isFloat16 = false); void readBinaryValueAndInsert(MutableColumnPtr column, NumpyDataType::Endianness endianness);
template <typename ColumnValue>
void readBinaryValueAndInsertFloat16(MutableColumnPtr column, NumpyDataType::Endianness endianness);
void readRows(MutableColumns & columns); void readRows(MutableColumns & columns);

3
tests/queries/0_stateless/02895_npy_format.reference
View File

@ -86,3 +86,6 @@ c
1 1
[2.199219,1.099609,3.300781] [2.199219,1.099609,3.300781]
[4.25,3.34961,6.628906] [4.25,3.34961,6.628906]
inf
nan
0

2
tests/queries/0_stateless/02895_npy_format.sh
View File

@ -58,3 +58,5 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy',
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS"
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/float_16.npy')" $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/float_16.npy')"
$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/npy_inf_nan_null.npy')"

BIN
tests/queries/0_stateless/data_npy/npy_inf_nan_null.npy Normal file
View File

Binary file not shown.