thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 00:52:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

NULLs support: fixed totally wrong code [#METR-19266].

Browse Source
This commit is contained in:
Alexey Milovidov 2016-12-30 08:13:14 +03:00
parent cefec8be5a
commit baf29f5c48
31 changed files with 334 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dbms/include/DB/Columns/ColumnNullable.h
View File

@ -57,8 +57,14 @@ public:
const ColumnPtr & getNestedColumn() const { return nested_column; }
/// Return the column that represents the byte map.
ColumnPtr & getNullValuesByteMap() { return null_map; }
const ColumnPtr & getNullValuesByteMap() const { return null_map; }
ColumnPtr & getNullMapColumn() { return null_map; }
const ColumnPtr & getNullMapColumn() const { return null_map; }
ColumnUInt8 & getNullMapConcreteColumn() { return static_cast<ColumnUInt8 &>(*null_map); }
const ColumnUInt8 & getNullMapConcreteColumn() const { return static_cast<const ColumnUInt8 &>(*null_map); }
ColumnUInt8::Container_t & getNullMap() { return getNullMapConcreteColumn().getData(); }
const ColumnUInt8::Container_t & getNullMap() const { return getNullMapConcreteColumn().getData(); }
/// Apply the null byte map of a specified nullable column onto the
/// null byte map of the current column by performing an element-wise OR
@ -67,11 +73,6 @@ public:
/// columns.
void applyNullValuesByteMap(const ColumnNullable & other);
private:
/// Convenience methods which make the implementation easier to read.
ColumnUInt8 & getNullMapContent() { return static_cast<ColumnUInt8 &>(*null_map); }
const ColumnUInt8 & getNullMapContent() const { return static_cast<const ColumnUInt8 &>(*null_map); }
private:
ColumnPtr nested_column;
ColumnPtr null_map;

6
dbms/include/DB/Common/StringUtils.h
View File

@ -121,3 +121,9 @@ inline char alternateCaseIfAlphaASCII(char c)
{
return c ^ 0x20;
}
inline bool equalsCaseInsensitive(char a, char b)
{
return a == b || (isAlphaASCII(a) && alternateCaseIfAlphaASCII(a) == b);
}

2
dbms/include/DB/DataStreams/NullableAdapterBlockInputStream.h
View File

@ -11,7 +11,7 @@ namespace DB
/// - if a target column is nullable while the corresponding source
/// column is not, we embed the source column into a nullable column;
/// - if a source column is nullable while the corresponding target
/// column is not, we extract the nested column from the source;
/// column is not, we extract the nested column from the source; /// TODO Probably this is totally incorrect.
/// - otherwise we just perform an identity mapping.
class NullableAdapterBlockInputStream : public IProfilingBlockInputStream
{

26
dbms/include/DB/DataTypes/DataTypeNull.h
View File

@ -1,13 +1,13 @@
#pragma once
#include <DB/DataTypes/IDataType.h>
#include <DB/DataTypes/NullSymbol.h>
#include <DB/Columns/ColumnConst.h>
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/WriteHelpers.h>
namespace DB
{
@ -21,7 +21,7 @@ public:
using FieldType = Null;
public:
std::string getName() const override
String getName() const override
{
return "Null";
}
@ -62,7 +62,7 @@ public:
void serializeBinary(const Field & field, WriteBuffer & ostr) const override
{
UInt8 x = 0;
UInt8 x = 1; /// Value is 1 to be consistent with NULLs serialization in DataTypeNullable.
writeBinary(x, ostr);
}
@ -75,7 +75,7 @@ public:
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
{
UInt8 x = 0;
UInt8 x = 1;
writeBinary(x, ostr);
}
@ -88,48 +88,48 @@ public:
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
{
writeCString(NullSymbol::Escaped::name, ostr);
writeCString("\\N", ostr);
}
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override
{
assertString(NullSymbol::Escaped::name, istr);
assertString("\\N", istr);
}
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
{
writeCString(NullSymbol::Quoted::name, ostr);
writeCString("NULL", ostr);
}
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override
{
assertString(NullSymbol::Quoted::name, istr);
assertStringCaseInsensitive("NULL", istr);
}
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
{
writeCString(NullSymbol::CSV::name, ostr);
writeCString("\\N", ostr);
}
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override
{
assertString(NullSymbol::CSV::name, istr);
assertString("\\N", istr);
}
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
{
writeCString(NullSymbol::Plain::name, ostr);
writeCString("NULL", ostr);
}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr,
bool force_quoting_64bit_integers) const override
{
writeCString(NullSymbol::JSON::name, ostr);
writeCString("null", ostr);
}
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override
{
assertString(NullSymbol::JSON::name, istr);
assertString("null", istr);
}
};

12
dbms/include/DB/DataTypes/DataTypeNullable.h
View File

@ -21,8 +21,10 @@ public:
DataTypePtr clone() const override { return std::make_shared<DataTypeNullable>(nested_data_type->clone()); }
/// Bulk serialization and deserialization is processing only nested columns. You should process null byte map separately.
void serializeBinary(const IColumn & column, WriteBuffer & ostr, size_t offset = 0, size_t limit = 0) const override;
void deserializeBinary(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
void serializeBinary(const Field & field, WriteBuffer & ostr) const override { nested_data_type->serializeBinary(field, ostr); }
void deserializeBinary(Field & field, ReadBuffer & istr) const override { nested_data_type->deserializeBinary(field, istr); }
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
@ -31,8 +33,18 @@ public:
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
/** It is questionable, how NULL values could be represented in CSV. There are three variants:
* 1. \N
* 2. empty string (without quotes)
* 3. NULL
* Now we support only first.
* In CSV, non-NULL string value, starting with \N characters, must be placed in quotes, to avoid ambiguity.
*/
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr,
bool force_quoting_64bit_integers) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;

43
dbms/include/DB/DataTypes/NullSymbol.h
View File

@ -1,43 +0,0 @@
#pragma once
#include <cstring>
namespace DB
{
namespace NullSymbol
{
struct Plain
{
static constexpr auto name = "NULL";
};
struct Escaped
{
static constexpr auto name = "\\N";
};
struct Quoted
{
static constexpr auto name = "NULL";
};
struct CSV
{
static constexpr auto name = "\\N";
};
struct JSON
{
static constexpr auto name = "null";
};
struct XML
{
static constexpr auto name = "\\N";
};
}
}

4
dbms/include/DB/Functions/FunctionsArray.h
View File

@ -1119,7 +1119,7 @@ public:
data.type = static_cast<const DataTypeNullable &>(*block.getByPosition(arguments[0]).type).getNestedType();
auto & null_map = source_block.unsafeGetByPosition(2);
null_map.column = nullable_col.getNullValuesByteMap();
null_map.column = nullable_col.getNullMapColumn();
null_map.type = std::make_shared<DataTypeUInt8>();
}
else
@ -1138,7 +1138,7 @@ public:
arg.type = static_cast<const DataTypeNullable &>(*block.getByPosition(arguments[1]).type).getNestedType();
auto & null_map = source_block.unsafeGetByPosition(3);
null_map.column = nullable_col.getNullValuesByteMap();
null_map.column = nullable_col.getNullMapColumn();
null_map.type = std::make_shared<DataTypeUInt8>();
}
else

2
dbms/include/DB/Functions/FunctionsConversion.h
View File

@ -2200,7 +2200,7 @@ private:
/// So we just keep the null map of the input argument.
const auto & col = block.getByPosition(arguments[0]).column;
const auto & nullable_col = static_cast<const ColumnNullable &>(*col);
null_map = nullable_col.getNullValuesByteMap();
null_map = nullable_col.getNullMapColumn();
}
else if (action & Action::CONVERT_NULL)
{

20
dbms/include/DB/Functions/FunctionsMiscellaneous.h
View File

@ -178,6 +178,11 @@ public:
return name;
}
bool hasSpecialSupportForNulls() const override
{
return true;
}
size_t getNumberOfArguments() const override { return 1; }
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
@ -207,6 +212,11 @@ public:
return name;
}
bool hasSpecialSupportForNulls() const override
{
return true;
}
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
@ -668,6 +678,8 @@ public:
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
bool hasSpecialSupportForNulls() const override { return true; }
String getName() const override { return name; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { return std::make_shared<DataTypeUInt8>(); }
@ -700,6 +712,8 @@ public:
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
bool hasSpecialSupportForNulls() const override { return true; }
String getName() const override { return name; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { return std::make_shared<DataTypeUInt8>(); }
@ -1197,9 +1211,9 @@ public:
IColumn & result_column = *result_column_ptr;
result_column.reserve(column_with_states->size());
auto arena = (agg_func.allocatesMemoryInArena()) ?
arenas_pool.get(0, []{ return new Arena(); }) :
nullptr;
auto arena = (agg_func.allocatesMemoryInArena())
? arenas_pool.get(0, []{ return new Arena(); })
: nullptr;
const auto & states = column_with_states->getData();
for (const auto & state_to_add : states)

30
dbms/include/DB/IO/ReadHelpers.h
View File

@ -180,6 +180,36 @@ inline bool checkChar(char c, ReadBuffer & buf)
return true;
}
bool checkStringCaseInsensitive(const char * s, ReadBuffer & buf);
inline bool checkStringCaseInsensitive(const String & s, ReadBuffer & buf)
{
return checkStringCaseInsensitive(s.c_str(), buf);
}
void assertStringCaseInsensitive(const char * s, ReadBuffer & buf);
inline void assertStringCaseInsensitive(const String & s, ReadBuffer & buf)
{
return assertStringCaseInsensitive(s.c_str(), buf);
}
/** Check that next character in buf matches first character of s.
* If true, then check all characters in s and throw exception if it doesn't match.
* If false, then return false, and leave position in buffer unchanged.
*/
bool checkStringByFirstCharacterAndAssertTheRest(const char * s, ReadBuffer & buf);
bool checkStringByFirstCharacterAndAssertTheRestCaseInsensitive(const char * s, ReadBuffer & buf);
inline bool checkStringByFirstCharacterAndAssertTheRest(const String & s, ReadBuffer & buf)
{
return checkStringByFirstCharacterAndAssertTheRest(s.c_str(), buf);
}
inline bool checkStringByFirstCharacterAndAssertTheRestCaseInsensitive(const String & s, ReadBuffer & buf)
{
return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive(s.c_str(), buf);
}
inline void readBoolText(bool & x, ReadBuffer & buf)
{
char tmp = '0';

2
dbms/include/DB/Interpreters/AggregationCommon.h
View File

@ -332,7 +332,7 @@ inline StringRef ALWAYS_INLINE extractKeysAndPlaceInPoolContiguous<true>(
{
const ColumnNullable & nullable_col = static_cast<const ColumnNullable &>(*key_columns[j]);
observed_column = nullable_col.getNestedColumn().get();
const auto & null_map = static_cast<const ColumnUInt8 &>(*nullable_col.getNullValuesByteMap()).getData();
const auto & null_map = nullable_col.getNullMap();
is_null = null_map[i] == 1;
}
else

8
dbms/include/DB/Interpreters/Aggregator.h
View File

@ -304,7 +304,7 @@ protected:
{
const auto & nullable_col = static_cast<const ColumnNullable &>(*col);
actual_columns.push_back(nullable_col.getNestedColumn().get());
null_maps.push_back(nullable_col.getNullValuesByteMap().get());
null_maps.push_back(nullable_col.getNullMapColumn().get());
}
else
{
@ -450,7 +450,7 @@ struct AggregationMethodKeysFixed
{
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(*key_columns[i]);
observed_column = nullable_col.getNestedColumn().get();
null_map = static_cast<ColumnUInt8 *>(nullable_col.getNullValuesByteMap().get());
null_map = static_cast<ColumnUInt8 *>(nullable_col.getNullMapColumn().get());
}
else
{
@ -597,7 +597,7 @@ private:
{
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(*key_columns[i]);
observed_column = nullable_col.getNestedColumn().get();
ColumnUInt8 & null_map = static_cast<ColumnUInt8 &>(*nullable_col.getNullValuesByteMap());
ColumnUInt8 & null_map = static_cast<ColumnUInt8 &>(*nullable_col.getNullMapColumn());
size_t bucket = i / 8;
size_t offset = i % 8;
@ -625,7 +625,7 @@ private:
{
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(*key_columns[i]);
observed_column = nullable_col.getNestedColumn().get();
null_map = static_cast<ColumnUInt8 *>(nullable_col.getNullValuesByteMap().get());
null_map = static_cast<ColumnUInt8 *>(nullable_col.getNullMapColumn().get());
}
else
{

2
dbms/include/DB/Interpreters/SetVariants.h
View File

@ -157,7 +157,7 @@ protected:
{
const auto & nullable_col = static_cast<const ColumnNullable &>(*col);
actual_columns.push_back(nullable_col.getNestedColumn().get());
null_maps.push_back(nullable_col.getNullValuesByteMap().get());
null_maps.push_back(nullable_col.getNullMapColumn().get());
}
else
{

135
dbms/src/Columns/ColumnNullable.cpp
View File

@ -17,6 +17,7 @@ ColumnNullable::ColumnNullable(ColumnPtr nested_column_, ColumnPtr null_map_)
throw Exception{"A nullable column cannot contain another nullable column", ErrorCodes::LOGICAL_ERROR};
}
ColumnPtr ColumnNullable::convertToFullColumnIfConst() const
{
ColumnPtr new_col_holder;
@ -27,25 +28,28 @@ ColumnPtr ColumnNullable::convertToFullColumnIfConst() const
return new_col_holder;
}
void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
{
const auto & arr = getNullMapContent().getData();
const auto & arr = getNullMap();
hash.update(reinterpret_cast<const char *>(&arr[n]), sizeof(arr[0]));
if (arr[n] == 0)
nested_column->updateHashWithValue(n, hash);
}
ColumnPtr ColumnNullable::cloneResized(size_t size) const
{
ColumnPtr new_nested_col = nested_column->cloneResized(size);
ColumnPtr new_null_map = getNullMapContent().cloneResized(size);
ColumnPtr new_null_map = getNullMapConcreteColumn().cloneResized(size); /// TODO Completely wrong.
return std::make_shared<ColumnNullable>(new_nested_col, new_null_map);
}
Field ColumnNullable::operator[](size_t n) const
{
if (isNullAt(n))
return Field{};
return Null();
else
{
const IColumn & col = *nested_column;
@ -53,10 +57,11 @@ Field ColumnNullable::operator[](size_t n) const
}
}
void ColumnNullable::get(size_t n, Field & res) const
{
if (isNullAt(n))
res = Field{};
res = Null();
else
nested_column->get(n, res);
}
@ -73,7 +78,7 @@ void ColumnNullable::insertData(const char * pos, size_t length)
StringRef ColumnNullable::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
{
const auto & arr = getNullMapContent().getData();
const auto & arr = getNullMap();
static constexpr auto s = sizeof(arr[0]);
auto pos = arena.allocContinue(s, begin);
@ -92,7 +97,7 @@ const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos)
UInt8 val = *reinterpret_cast<const UInt8 *>(pos);
pos += sizeof(val);
getNullMapContent().insert(val);
getNullMap().push_back(val);
if (val == 0)
pos = nested_column->deserializeAndInsertFromArena(pos);
@ -105,7 +110,7 @@ const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos)
void ColumnNullable::insertRangeFrom(const IColumn & src, size_t start, size_t length)
{
const ColumnNullable & nullable_col = static_cast<const ColumnNullable &>(src);
getNullMapContent().insertRangeFrom(*nullable_col.null_map, start, length);
getNullMapConcreteColumn().insertRangeFrom(*nullable_col.null_map, start, length);
nested_column->insertRangeFrom(*nullable_col.nested_column, start, length);
}
@ -114,38 +119,38 @@ void ColumnNullable::insert(const Field & x)
if (x.isNull())
{
nested_column->insertDefault();
getNullMapContent().insert(1);
getNullMap().push_back(1);
}
else
{
nested_column->insert(x);
getNullMapContent().insert(0);
getNullMap().push_back(0);
}
}
void ColumnNullable::insertDefault()
{
nested_column->insertDefault();
getNullMapContent().insert(1);
getNullMap().push_back(1);
}
void ColumnNullable::popBack(size_t n)
{
nested_column->popBack(n);
getNullMapContent().popBack(n);
getNullMapConcreteColumn().popBack(n);
}
ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const
{
ColumnPtr filtered_data = nested_column->filter(filt, result_size_hint);
ColumnPtr filtered_null_map = getNullMapContent().filter(filt, result_size_hint);
ColumnPtr filtered_null_map = getNullMapConcreteColumn().filter(filt, result_size_hint);
return std::make_shared<ColumnNullable>(filtered_data, filtered_null_map);
}
ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const
{
ColumnPtr permuted_data = nested_column->permute(perm, limit);
ColumnPtr permuted_null_map = getNullMapContent().permute(perm, limit);
ColumnPtr permuted_null_map = getNullMapConcreteColumn().permute(perm, limit);
return std::make_shared<ColumnNullable>(permuted_data, permuted_null_map);
}
@ -242,12 +247,12 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, Permutation & re
void ColumnNullable::reserve(size_t n)
{
nested_column->reserve(n);
getNullMapContent().reserve(n);
getNullMap().reserve(n);
}
size_t ColumnNullable::byteSize() const
{
return nested_column->byteSize() + getNullMapContent().byteSize();
return nested_column->byteSize() + getNullMapConcreteColumn().byteSize();
}
namespace
@ -264,87 +269,105 @@ void getExtremesFromNullableContent(const ColumnVector<T> & col, const NullValue
if (size == 0)
{
min = typename NearestFieldType<T>::Type(0);
max = typename NearestFieldType<T>::Type(0);
min = Null();
max = Null();
return;
}
size_t min_i = 0;
bool has_not_null = false;
bool has_not_nan = false;
for (; min_i < size; ++min_i)
T cur_min = 0;
T cur_max = 0;
for (size_t i = 0; i < size; ++i)
{
if (null_map[min_i] == 0)
break;
}
const T x = data[i];
if (min_i == size)
{
min = Field{};
max = Field{};
return;
}
T cur_min = data[min_i];
T cur_max = data[min_i];
for (size_t i = min_i + 1; i < size; ++i)
{
if (null_map[i] != 0)
if (null_map[i])
continue;
if (data[i] < cur_min)
cur_min = data[i];
if (data[i] > cur_max)
cur_max = data[i];
if (!has_not_null)
{
cur_min = x;
cur_max = x;
has_not_null = true;
continue;
}
if (isNaN(x))
continue;
if (!has_not_nan)
{
cur_min = x;
cur_max = x;
has_not_nan = true;
continue;
}
if (x < cur_min)
cur_min = x;
if (x > cur_max)
cur_max = x;
}
if (has_not_null)
{
min = typename NearestFieldType<T>::Type(cur_min);
max = typename NearestFieldType<T>::Type(cur_max);
}
}
}
void ColumnNullable::getExtremes(Field & min, Field & max) const
{
min = Null();
max = Null();
const auto & null_map = getNullMap();
if (const auto col = typeid_cast<const ColumnInt8 *>(nested_column.get()))
getExtremesFromNullableContent<Int8>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<Int8>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnInt16 *>(nested_column.get()))
getExtremesFromNullableContent<Int16>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<Int16>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnInt32 *>(nested_column.get()))
getExtremesFromNullableContent<Int32>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<Int32>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnInt64 *>(nested_column.get()))
getExtremesFromNullableContent<Int64>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<Int64>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnUInt8 *>(nested_column.get()))
getExtremesFromNullableContent<UInt8>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<UInt8>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnUInt16 *>(nested_column.get()))
getExtremesFromNullableContent<UInt16>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<UInt16>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnUInt32 *>(nested_column.get()))
getExtremesFromNullableContent<UInt32>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<UInt32>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnUInt64 *>(nested_column.get()))
getExtremesFromNullableContent<UInt64>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<UInt64>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnFloat32 *>(nested_column.get()))
getExtremesFromNullableContent<Float32>(*col, getNullMapContent().getData(), min, max);
getExtremesFromNullableContent<Float32>(*col, null_map, min, max);
else if (const auto col = typeid_cast<const ColumnFloat64 *>(nested_column.get()))
getExtremesFromNullableContent<Float64>(*col, getNullMapContent().getData(), min, max);
else
nested_column->getExtremes(min, max);
getExtremesFromNullableContent<Float64>(*col, null_map, min, max);
}
ColumnPtr ColumnNullable::replicate(const Offsets_t & offsets) const
{
ColumnPtr replicated_data = nested_column->replicate(offsets);
ColumnPtr replicated_null_map = getNullMapContent().replicate(offsets);
ColumnPtr replicated_null_map = getNullMapConcreteColumn().replicate(offsets);
return std::make_shared<ColumnNullable>(replicated_data, replicated_null_map);
}
void ColumnNullable::applyNullValuesByteMap(const ColumnNullable & other)
{
NullValuesByteMap & arr1 = getNullMapContent().getData();
const NullValuesByteMap & arr2 = other.getNullMapContent().getData();
NullValuesByteMap & arr1 = getNullMap();
const NullValuesByteMap & arr2 = other.getNullMap();
if (arr1.size() != arr2.size())
throw Exception{"Inconsistent sizes", ErrorCodes::LOGICAL_ERROR};
throw Exception{"Inconsistent sizes of ColumnNullable objects", ErrorCodes::LOGICAL_ERROR};
for (size_t i = 0; i < arr1.size(); ++i)
arr1[i] |= arr2[i];

2
dbms/src/DataStreams/FilterBlockInputStream.cpp
View File

@ -171,7 +171,7 @@ Block FilterBlockInputStream::readImpl()
auto & filter_col = actual_col.getData();
/// Access the null values byte map content.
ColumnPtr & null_map = nullable_col.getNullValuesByteMap();
ColumnPtr & null_map = nullable_col.getNullMapColumn();
ColumnUInt8 & content = static_cast<ColumnUInt8 &>(*null_map);
auto & data = content.getData();

2
dbms/src/DataStreams/NativeBlockInputStream.cpp
View File

@ -54,7 +54,7 @@ void NativeBlockInputStream::readData(const IDataType & type, IColumn & column,
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(column);
IColumn & nested_col = *nullable_col.getNestedColumn();
IColumn & null_map = *nullable_col.getNullValuesByteMap();
IColumn & null_map = *nullable_col.getNullMapColumn();
DataTypeUInt8{}.deserializeBinary(null_map, istr, rows, 0);
readData(nested_type, nested_col, istr, rows);

2
dbms/src/DataStreams/NativeBlockOutputStream.cpp
View File

@ -61,7 +61,7 @@ void NativeBlockOutputStream::writeData(const IDataType & type, const ColumnPtr
const ColumnNullable & nullable_col = static_cast<const ColumnNullable &>(*full_column.get());
const ColumnPtr & nested_col = nullable_col.getNestedColumn();
const IColumn & null_map = *nullable_col.getNullValuesByteMap();
const IColumn & null_map = *nullable_col.getNullMapColumn();
DataTypeUInt8{}.serializeBinary(null_map, ostr, offset, limit);
writeData(nested_type, nested_col, ostr, offset, limit);

2
dbms/src/DataStreams/NullableAdapterBlockInputStream.cpp
View File

@ -52,7 +52,7 @@ Block NullableAdapterBlockInputStream::readImpl()
const auto & nullable_col = static_cast<const ColumnNullable &>(*elem.column);
const auto & nullable_type = static_cast<const DataTypeNullable &>(*elem.type);
const auto & null_map = static_cast<const ColumnUInt8 &>(*nullable_col.getNullValuesByteMap()).getData();
const auto & null_map = nullable_col.getNullMap();
bool has_nulls = std::any_of(null_map.begin(), null_map.end(), [](UInt8 val){ return val == 1; });
if (has_nulls)

2
dbms/src/DataTypes/DataTypeNull.cpp
View File

@ -10,7 +10,7 @@ void DataTypeNull::serializeBinary(const IColumn & column, WriteBuffer & ostr, s
if ((limit == 0) || ((offset + limit) > size))
limit = size - offset;
UInt8 x = 0;
UInt8 x = 1;
writeBinary(x, limit, ostr);
}

230
dbms/src/DataTypes/DataTypeNullable.cpp
View File

@ -1,97 +1,15 @@
#include <DB/DataTypes/DataTypeNullable.h>
#include <DB/DataTypes/NullSymbol.h>
#include <DB/Columns/ColumnNullable.h>
#include <DB/IO/ReadBuffer.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteBuffer.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/ConcatReadBuffer.h>
namespace DB
{
namespace
{
/// When a column is serialized as a binary data file, its null values are directly
/// represented as null symbols into this file. The methods below are helpers that
/// we use for the binary deserialization. They are used as follows:
///
/// auto action = NullDeserializer<NullSymbol::XXX>::execute(col, istr);
///
/// if (action == Action::NONE) /// eof
/// return;
/// else if (action == Action::ADD_ORDINARY) /// add an ordinary value
/// ... deserialize the nested column ...
///
/// updateNullMap(col, action);
///
/// This two-step process is required because when we perform an INSERT query
/// whose values are expressions, ValuesRowInputStream attempts to deserialize
/// a stream, then raises an exception, and finally evaluates expressions inside
/// the exception handler. If we did something like:
///
/// ... deserialize the null map...
/// ... deserialize the nested column...
///
/// there would be garbage in the null map.
/// Action to be performed while updating the null map of a nullable column.
enum class Action
{
NONE, /// do nothing
ADD_NULL, /// add a value indicating a NULL
ADD_ORDINARY /// add a value indicating an ordinary value
};
/// The template class below provides one method that takes a nullable column being
/// deserialized and looks if there is a pending null symbol in the corresponding
/// binary data file. It returns the appropriate action to be performed on the null
/// map of the column.
template <typename Null>
struct NullDeserializer
{
static Action execute(ColumnNullable & col, ReadBuffer & istr)
{
if (!istr.eof())
{
if (*istr.position() == Null::name[0])
{
++istr.position();
static constexpr auto length = __builtin_strlen(Null::name);
if (length > 1)
assertString(&Null::name[1], istr);
return Action::ADD_NULL;
}
else
return Action::ADD_ORDINARY;
}
else
return Action::NONE;
}
};
/// This function takes the appropiate action when updating the null map of a nullable
/// column.
void updateNullMap(ColumnNullable & col, const Action & action)
{
auto & null_map = static_cast<ColumnUInt8 &>(*col.getNullValuesByteMap()).getData();
if (action == Action::ADD_NULL)
{
null_map.push_back(1);
ColumnPtr & nested_col = col.getNestedColumn();
nested_col->insertDefault();
}
else if (action == Action::ADD_ORDINARY)
null_map.push_back(0);
else
throw Exception{"DataTypeNullable: internal error", ErrorCodes::LOGICAL_ERROR};
}
}
DataTypeNullable::DataTypeNullable(DataTypePtr nested_data_type_)
: nested_data_type{nested_data_type_}
@ -110,40 +28,108 @@ void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr, si
nested_data_type->deserializeBinary(*col.getNestedColumn(), istr, limit, avg_value_size_hint);
}
void DataTypeNullable::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
{
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
bool is_null = col.isNullAt(row_num);
writeBinary(is_null, ostr);
if (!is_null)
nested_data_type->serializeBinary(*col.getNestedColumn(), row_num, ostr);
}
void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr) const
/// We need to insert both to nested column and to null byte map, or, in case of exception, to not insert at all.
template <typename CheckForNull, typename DeserializeNested>
static void safeDeserialize(
IColumn & column,
CheckForNull && check_for_null, DeserializeNested && deserialize_nested)
{
ColumnNullable & col = static_cast<ColumnNullable &>(column);
nested_data_type->deserializeBinary(*col.getNestedColumn(), istr);
if (check_for_null())
{
col.insertDefault();
}
else
{
deserialize_nested(*col.getNestedColumn());
try
{
col.getNullMap().push_back(0);
}
catch (...)
{
col.getNestedColumn()->popBack(1);
throw;
}
}
}
void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr) const
{
safeDeserialize(column,
[&istr] { bool is_null = 0; readBinary(is_null, istr); return is_null; },
[this, &istr] (IColumn & nested) { nested_data_type->deserializeBinary(nested, istr); } );
}
void DataTypeNullable::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
{
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
writeCString(NullSymbol::Escaped::name, ostr);
writeCString("\\N", ostr);
else
nested_data_type->serializeTextEscaped(*col.getNestedColumn(), row_num, ostr);
}
void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
{
ColumnNullable & col = static_cast<ColumnNullable &>(column);
/// Little tricky, because we cannot discriminate null from first character.
auto action = NullDeserializer<NullSymbol::Escaped>::execute(col, istr);
if (istr.eof())
throw Exception("Unexpected end of stream, while parsing value of Nullable type", ErrorCodes::CANNOT_READ_ALL_DATA);
if (action == Action::NONE)
return;
else if (action == Action::ADD_ORDINARY)
nested_data_type->deserializeTextEscaped(*col.getNestedColumn(), istr);
/// This is not null, surely.
if (*istr.position() != '\\')
{
safeDeserialize(column,
[&istr] { return false; },
[this, &istr] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr); } );
}
else
{
/// Now we know, that data in buffer starts with backslash.
++istr.position();
updateNullMap(col, action);
if (istr.eof())
throw Exception("Unexpected end of stream, while parsing value of Nullable type, after backslash", ErrorCodes::CANNOT_READ_ALL_DATA);
safeDeserialize(column,
[&istr]
{
if (*istr.position() == 'N')
{
++istr.position();
return true;
}
return false;
},
[this, &istr] (IColumn & nested)
{
/// We need to place backslash back in front of istr.
ReadBuffer prefix(const_cast<char *>("\\"), 1, 0);
ConcatReadBuffer prepended_istr(prefix, istr);
nested_data_type->deserializeTextQuoted(nested, prepended_istr);
});
}
}
void DataTypeNullable::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
@ -151,29 +137,17 @@ void DataTypeNullable::serializeTextQuoted(const IColumn & column, size_t row_nu
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
{
/// This is not a typo. We really mean "Escaped" and not "Quoted".
/// The reason is that, when displaying an array of nullable strings,
/// we want to see \N instead of NULL.
writeCString(NullSymbol::Escaped::name, ostr);
}
writeCString("NULL", ostr);
else
nested_data_type->serializeTextQuoted(*col.getNestedColumn(), row_num, ostr);
}
void DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
{
ColumnNullable & col = static_cast<ColumnNullable &>(column);
auto action = NullDeserializer<NullSymbol::Quoted>::execute(col, istr);
if (action == Action::NONE)
return;
else if (action == Action::ADD_ORDINARY)
nested_data_type->deserializeTextQuoted(*col.getNestedColumn(), istr);
updateNullMap(col, action);
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
[this, &istr] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr); } );
}
void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
@ -181,23 +155,16 @@ void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num,
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
writeCString(NullSymbol::Quoted::name, ostr);
writeCString("\\N", ostr);
else
nested_data_type->serializeTextCSV(*col.getNestedColumn(), row_num, ostr);
}
void DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
{
ColumnNullable & col = static_cast<ColumnNullable &>(column);
auto action = NullDeserializer<NullSymbol::Quoted>::execute(col, istr);
if (action == Action::NONE)
return;
else if (action == Action::ADD_ORDINARY)
nested_data_type->deserializeTextCSV(*col.getNestedColumn(), istr, delimiter);
updateNullMap(col, action);
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRest("\\N", istr); },
[this, &istr] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr); } );
}
void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
@ -205,7 +172,7 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
writeCString(NullSymbol::Plain::name, ostr);
writeCString("NULL", ostr);
else
nested_data_type->serializeText(*col.getNestedColumn(), row_num, ostr);
}
@ -216,7 +183,7 @@ void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num,
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
writeCString(NullSymbol::JSON::name, ostr);
writeCString("null", ostr);
else
nested_data_type->serializeTextJSON(*col.getNestedColumn(), row_num, ostr,
force_quoting_64bit_integers);
@ -224,16 +191,9 @@ void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num,
void DataTypeNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
{
ColumnNullable & col = static_cast<ColumnNullable &>(column);
auto action = NullDeserializer<NullSymbol::JSON>::execute(col, istr);
if (action == Action::NONE)
return;
else if (action == Action::ADD_ORDINARY)
nested_data_type->deserializeTextJSON(*col.getNestedColumn(), istr);
updateNullMap(col, action);
safeDeserialize(column,
[&istr] { return checkStringByFirstCharacterAndAssertTheRest("null", istr); },
[this, &istr] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr); } );
}
void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
@ -241,7 +201,7 @@ void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num,
const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
if (col.isNullAt(row_num))
writeCString(NullSymbol::XML::name, ostr);
writeCString("\\N", ostr);
else
nested_data_type->serializeTextXML(*col.getNestedColumn(), row_num, ostr);
}

18
dbms/src/DataTypes/NullSymbol.cpp
View File

@ -1,18 +0,0 @@
#include <DB/DataTypes/NullSymbol.h>
namespace DB
{
namespace NullSymbol
{
constexpr decltype(Plain::name) Plain::name;
constexpr decltype(Escaped::name) Escaped::name;
constexpr decltype(Quoted::name) Quoted::name;
constexpr decltype(CSV::name) CSV::name;
constexpr decltype(JSON::name) JSON::name;
constexpr decltype(XML::name) XML::name;
}
}

2
dbms/src/Functions/Conditional/CondSource.cpp
View File

@ -94,7 +94,7 @@ const PaddedPODArray<UInt8> & CondSource::initNullMap(const Block & block, const
if (col->isNullable())
{
const ColumnNullable & nullable_col = static_cast<const ColumnNullable &>(*col);
const ColumnPtr & null_map = nullable_col.getNullValuesByteMap();
const ColumnPtr & null_map = nullable_col.getNullMapColumn();
const ColumnUInt8 & content = static_cast<const ColumnUInt8 &>(*null_map);
return content.getData();

2
dbms/src/Functions/Conditional/NullMapBuilder.cpp
View File

@ -66,7 +66,7 @@ namespace Conditional
else if (from.isNullable())
{
const auto & nullable_col = static_cast<const ColumnNullable &>(from);
null_map = nullable_col.getNullValuesByteMap();
null_map = nullable_col.getNullMapColumn();
}
else
null_map = std::make_shared<ColumnUInt8>(row_count, 0);

22
dbms/src/Functions/FunctionsArray.cpp
View File

@ -1321,7 +1321,7 @@ void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & argumen
has_nullable_columns = true;
const auto & nullable_col = static_cast<const ColumnNullable &>(*data_columns[i]);
data_columns[i] = nullable_col.getNestedColumn().get();
null_maps[i] = nullable_col.getNullValuesByteMap().get();
null_maps[i] = nullable_col.getNullMapColumn().get();
}
else
null_maps[i] = nullptr;
@ -1645,7 +1645,7 @@ void FunctionArrayEnumerateUniq::executeImpl(Block & block, const ColumnNumbers
has_nullable_columns = true;
const auto & nullable_col = static_cast<const ColumnNullable &>(*data_columns[i]);
data_columns[i] = nullable_col.getNestedColumn().get();
null_maps[i] = nullable_col.getNullValuesByteMap().get();
null_maps[i] = nullable_col.getNullMapColumn().get();
}
else
null_maps[i] = nullptr;
@ -2001,7 +2001,7 @@ bool FunctionEmptyArrayToSingle::executeNumber(
res_data.reserve(src_data.size());
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
nullable_res_col->getNullValuesByteMap() = nullable_col->getNullValuesByteMap();
nullable_res_col->getNullMapColumn() = nullable_col->getNullMapColumn();
ColumnArray::Offset_t src_prev_offset = 0;
ColumnArray::Offset_t res_prev_offset = 0;
@ -2054,7 +2054,7 @@ bool FunctionEmptyArrayToSingle::executeFixedString(
res_data.reserve(src_data.size());
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
nullable_res_col->getNullValuesByteMap() = nullable_col->getNullValuesByteMap();
nullable_res_col->getNullMapColumn() = nullable_col->getNullMapColumn();
ColumnArray::Offset_t src_prev_offset = 0;
ColumnArray::Offset_t res_prev_offset = 0;
@ -2116,7 +2116,7 @@ bool FunctionEmptyArrayToSingle::executeString(
res_data.reserve(src_data.size());
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
nullable_res_col->getNullValuesByteMap() = nullable_col->getNullValuesByteMap();
nullable_res_col->getNullMapColumn() = nullable_col->getNullMapColumn();
ColumnArray::Offset_t src_array_prev_offset = 0;
ColumnArray::Offset_t res_array_prev_offset = 0;
@ -2442,8 +2442,8 @@ bool FunctionArrayReverse::executeNumber(
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
{
/// Make a reverted null map.
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullValuesByteMap()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullValuesByteMap()).getData();
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullMapColumn()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullMapColumn()).getData();
res_null_map.resize(src_data.size());
do_reverse(src_null_map, src_offsets, res_null_map);
}
@ -2494,8 +2494,8 @@ bool FunctionArrayReverse::executeFixedString(
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
{
/// Make a reverted null map.
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullValuesByteMap()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullValuesByteMap()).getData();
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullMapColumn()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullMapColumn()).getData();
res_null_map.resize(src_null_map.size());
ColumnArray::Offset_t src_prev_offset = 0;
@ -2574,8 +2574,8 @@ bool FunctionArrayReverse::executeString(
if ((nullable_col != nullptr) && (nullable_res_col != nullptr))
{
/// Make a reverted null map.
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullValuesByteMap()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullValuesByteMap()).getData();
const auto & src_null_map = static_cast<const ColumnUInt8 &>(*nullable_col->getNullMapColumn()).getData();
auto & res_null_map = static_cast<ColumnUInt8 &>(*nullable_res_col->getNullMapColumn()).getData();
res_null_map.resize(src_string_offsets.size());
size_t size = src_string_offsets.size();

1
dbms/src/Functions/FunctionsMiscellaneous.cpp
View File

@ -5,7 +5,6 @@
#include <DB/Functions/FunctionsMiscellaneous.h>
#include <DB/Functions/DataTypeTraits.h>
#include <DB/DataTypes/DataTypeEnum.h>
#include <DB/DataTypes/NullSymbol.h>
#include <DB/DataTypes/DataTypeNullable.h>
#include <DB/Columns/ColumnNullable.h>
#include <common/ClickHouseRevision.h>

2
dbms/src/Functions/FunctionsNull.cpp
View File

@ -56,7 +56,7 @@ void FunctionIsNull::executeImpl(Block & block, const ColumnNumbers & arguments,
{
/// Merely return the embedded null map.
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(*elem.column);
block.getByPosition(result).column = nullable_col.getNullValuesByteMap();
block.getByPosition(result).column = nullable_col.getNullMapColumn();
}
else
{

39
dbms/src/IO/ReadHelpers.cpp
View File

@ -50,7 +50,7 @@ bool checkString(const char * s, ReadBuffer & buf)
}
static bool checkStringCaseInsensitive(const char * s, ReadBuffer & buf)
bool checkStringCaseInsensitive(const char * s, ReadBuffer & buf)
{
for (; *s; ++s)
{
@ -58,7 +58,7 @@ static bool checkStringCaseInsensitive(const char * s, ReadBuffer & buf)
return false;
char c = *buf.position();
if (!(*s == c || (isAlphaASCII(*s) && alternateCaseIfAlphaASCII(*s) == c)))
if (!equalsCaseInsensitive(*s, c))
return false;
++buf.position();
@ -90,6 +90,36 @@ void assertEOF(ReadBuffer & buf)
}
void assertStringCaseInsensitive(const char * s, ReadBuffer & buf)
{
if (!checkStringCaseInsensitive(s, buf))
throwAtAssertionFailed(s, buf);
}
bool checkStringByFirstCharacterAndAssertTheRest(const char * s, ReadBuffer & buf)
{
if (buf.eof() || *buf.position() != *s)
return false;
assertString(s, buf);
return true;
}
bool checkStringByFirstCharacterAndAssertTheRestCaseInsensitive(const char * s, ReadBuffer & buf)
{
if (buf.eof())
return false;
char c = *buf.position();
if (!equalsCaseInsensitive(*s, c))
return false;
assertStringCaseInsensitive(s, buf);
return true;
}
template <typename T>
static void appendToStringOrVector(T & s, const char * begin, const char * end)
{
@ -174,6 +204,11 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf)
readPODBinary(c2, buf);
s.push_back(static_cast<char>(unhex(c1) * 16 + unhex(c2)));
}
else if (*buf.position() == 'N')
{
/// Support for NULLs: \N sequence must be parsed as empty string.
++buf.position();
}
else
{
/// Обычная escape-последовательность из одного символа.

2
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
View File

@ -416,7 +416,7 @@ void MergeTreeReader::readData(const String & name, const IDataType & type, ICol
Stream & stream = *(streams.at(filename));
stream.seekToMark(from_mark);
IColumn & col8 = *(nullable_col.getNullValuesByteMap());
IColumn & col8 = *(nullable_col.getNullMapColumn());
DataTypeUInt8{}.deserializeBinary(col8, *stream.data_buffer, max_rows_to_read, 0);
/// Then read data.

2
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
View File

@ -170,7 +170,7 @@ void IMergedBlockOutputStream::writeDataImpl(
writeIntBinary(stream.compressed.offset(), stream.marks);
}
DataTypeUInt8{}.serializeBinary(*(nullable_col.getNullValuesByteMap()), stream.compressed);
DataTypeUInt8{}.serializeBinary(*(nullable_col.getNullMapColumn()), stream.compressed);
/// Чтобы вместо засечек, указывающих на конец сжатого блока, были засечки, указывающие на начало следующего.
stream.compressed.nextIfAtEnd();

4
dbms/src/Storages/StorageLog.cpp
View File

@ -359,7 +359,7 @@ void LogBlockInputStream::readData(const String & name, const IDataType & type,
ColumnNullable & nullable_col = static_cast<ColumnNullable &>(column);
IColumn & nested_col = *nullable_col.getNestedColumn();
DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullValuesByteMap(),
DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullMapColumn(),
streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed, max_rows_to_read, 0);
/// Then read data.
readData(name, nested_type, nested_col, max_rows_to_read, level, read_offsets);
@ -490,7 +490,7 @@ void LogBlockOutputStream::writeData(const String & name, const IDataType & type
out_null_marks.emplace_back(storage.files[filename].column_index, mark);
DataTypeUInt8{}.serializeBinary(*nullable_col.getNullValuesByteMap(),
DataTypeUInt8{}.serializeBinary(*nullable_col.getNullMapColumn(),
streams[filename]->compressed);
streams[filename]->compressed.next();

4
dbms/src/Storages/StorageTinyLog.cpp
View File

@ -300,7 +300,7 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty
IColumn & nested_col = *nullable_col.getNestedColumn();
/// First read from the null map.
DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullValuesByteMap(),
DataTypeUInt8{}.deserializeBinary(*nullable_col.getNullMapColumn(),
streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed, limit, 0);
/// Then read data.
@ -372,7 +372,7 @@ void TinyLogBlockOutputStream::writeData(const String & name, const IDataType &
const ColumnNullable & nullable_col = static_cast<const ColumnNullable &>(column);
const IColumn & nested_col = *nullable_col.getNestedColumn();
DataTypeUInt8{}.serializeBinary(*nullable_col.getNullValuesByteMap(),
DataTypeUInt8{}.serializeBinary(*nullable_col.getNullMapColumn(),
streams[name + DBMS_STORAGE_LOG_DATA_BINARY_NULL_MAP_EXTENSION]->compressed);
/// Then write data.