diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 03327f76e48..b103f42fcc5 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -459,6 +459,8 @@ public: explicit FieldVisitorMax(const Field & rhs_) : rhs(rhs_) {} bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } + bool operator() (NegativeInfinity &) const { throw Exception("Cannot compare -Inf", ErrorCodes::LOGICAL_ERROR); } + bool operator() (PositiveInfinity &) const { throw Exception("Cannot compare +Inf", ErrorCodes::LOGICAL_ERROR); } bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot compare AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } bool operator() (Array & x) const { return compareImpl(x); } @@ -494,6 +496,8 @@ public: explicit FieldVisitorMin(const Field & rhs_) : rhs(rhs_) {} bool operator() (Null &) const { throw Exception("Cannot compare Nulls", ErrorCodes::LOGICAL_ERROR); } + bool operator() (NegativeInfinity &) const { throw Exception("Cannot compare -Inf", ErrorCodes::LOGICAL_ERROR); } + bool operator() (PositiveInfinity &) const { throw Exception("Cannot compare +Inf", ErrorCodes::LOGICAL_ERROR); } bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } bool operator() (Array & x) const { return compareImpl(x); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 62524315354..dec93fc7a30 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -546,97 +546,54 @@ namespace { /// The following function implements a slightly more general version -/// of getExtremes() than the implementation from ColumnVector. +/// of getExtremes() than the implementation from Not-Null IColumns. /// It takes into account the possible presence of nullable values. -template -void getExtremesFromNullableContent(const ColumnVector & col, const NullMap & null_map, Field & min, Field & max) +void getExtremesWithNulls(const IColumn & nested_column, const NullMap & null_array, Field & min, Field & max, bool null_last = false) { - const auto & data = col.getData(); - size_t size = data.size(); - - if (size == 0) + size_t number_of_nulls = 0; + size_t n = null_array.size(); + NullMap not_null_array(n); + for (auto i = 0ul; i < n; ++i) { - min = Null(); - max = Null(); - return; - } - - bool has_not_null = false; - bool has_not_nan = false; - - T cur_min = 0; - T cur_max = 0; - - for (size_t i = 0; i < size; ++i) - { - const T x = data[i]; - - if (null_map[i]) - continue; - - if (!has_not_null) + if (null_array[i]) { - cur_min = x; - cur_max = x; - has_not_null = true; - has_not_nan = !isNaN(x); - continue; + ++number_of_nulls; + not_null_array[i] = 0; } - - if (isNaN(x)) - continue; - - if (!has_not_nan) + else { - cur_min = x; - cur_max = x; - has_not_nan = true; - continue; + not_null_array[i] = 1; } - - if (x < cur_min) - cur_min = x; - else if (x > cur_max) - cur_max = x; } - - if (has_not_null) + if (number_of_nulls == 0) { - min = cur_min; - max = cur_max; + nested_column.getExtremes(min, max); + } + else if (number_of_nulls == n) + { + min = PositiveInfinity(); + max = PositiveInfinity(); + } + else + { + auto filtered_column = nested_column.filter(not_null_array, -1); + filtered_column->getExtremes(min, max); + if (null_last) + max = PositiveInfinity(); } } - } void ColumnNullable::getExtremes(Field & min, Field & max) const { - min = Null(); - max = Null(); + getExtremesWithNulls(getNestedColumn(), getNullMapData(), min, max); +} - const auto & null_map_data = getNullMapData(); - if (const auto * col_i8 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_i8, null_map_data, min, max); - else if (const auto * col_i16 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_i16, null_map_data, min, max); - else if (const auto * col_i32 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_i32, null_map_data, min, max); - else if (const auto * col_i64 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_i64, null_map_data, min, max); - else if (const auto * col_u8 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_u8, null_map_data, min, max); - else if (const auto * col_u16 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_u16, null_map_data, min, max); - else if (const auto * col_u32 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_u32, null_map_data, min, max); - else if (const auto * col_u64 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_u64, null_map_data, min, max); - else if (const auto * col_f32 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_f32, null_map_data, min, max); - else if (const auto * col_f64 = typeid_cast(nested_column.get())) - getExtremesFromNullableContent(*col_f64, null_map_data, min, max); +void ColumnNullable::getExtremesNullLast(Field & min, Field & max) const +{ + getExtremesWithNulls(getNestedColumn(), getNullMapData(), min, max, true); } diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 963b3e1e8fa..7b339893ff4 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -111,6 +111,8 @@ public: void updateWeakHash32(WeakHash32 & hash) const override; void updateHashFast(SipHash & hash) const override; void getExtremes(Field & min, Field & max) const override; + // Special function for nullable minmax index + void getExtremesNullLast(Field & min, Field & max) const; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override { diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h index 0f099c6215d..82a804691d7 100644 --- a/src/Common/FieldVisitorConvertToNumber.h +++ b/src/Common/FieldVisitorConvertToNumber.h @@ -26,6 +26,16 @@ public: throw Exception("Cannot convert NULL to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); } + T operator() (const NegativeInfinity &) const + { + throw Exception("Cannot convert -Inf to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + + T operator() (const PositiveInfinity &) const + { + throw Exception("Cannot convert +Inf to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); + } + T operator() (const String &) const { throw Exception("Cannot convert String to " + demangle(typeid(T).name()), ErrorCodes::CANNOT_CONVERT_TYPE); diff --git a/src/Common/FieldVisitorDump.cpp b/src/Common/FieldVisitorDump.cpp index e6726a4502e..5e767cf30c1 100644 --- a/src/Common/FieldVisitorDump.cpp +++ b/src/Common/FieldVisitorDump.cpp @@ -25,6 +25,8 @@ static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) } String FieldVisitorDump::operator() (const Null &) const { return "NULL"; } +String FieldVisitorDump::operator() (const NegativeInfinity &) const { return "-Inf"; } +String FieldVisitorDump::operator() (const PositiveInfinity &) const { return "+Inf"; } String FieldVisitorDump::operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); } String FieldVisitorDump::operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); } String FieldVisitorDump::operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); } diff --git a/src/Common/FieldVisitorDump.h b/src/Common/FieldVisitorDump.h index 22e34d66ff7..bc82d35f0f1 100644 --- a/src/Common/FieldVisitorDump.h +++ b/src/Common/FieldVisitorDump.h @@ -10,6 +10,8 @@ class FieldVisitorDump : public StaticVisitor { public: String operator() (const Null & x) const; + String operator() (const NegativeInfinity & x) const; + String operator() (const PositiveInfinity & x) const; String operator() (const UInt64 & x) const; String operator() (const UInt128 & x) const; String operator() (const UInt256 & x) const; diff --git a/src/Common/FieldVisitorHash.cpp b/src/Common/FieldVisitorHash.cpp index 80d5f2daf65..259dd871d20 100644 --- a/src/Common/FieldVisitorHash.cpp +++ b/src/Common/FieldVisitorHash.cpp @@ -14,6 +14,18 @@ void FieldVisitorHash::operator() (const Null &) const hash.update(type); } +void FieldVisitorHash::operator() (const NegativeInfinity &) const +{ + UInt8 type = Field::Types::NegativeInfinity; + hash.update(type); +} + +void FieldVisitorHash::operator() (const PositiveInfinity &) const +{ + UInt8 type = Field::Types::PositiveInfinity; + hash.update(type); +} + void FieldVisitorHash::operator() (const UInt64 & x) const { UInt8 type = Field::Types::UInt64; diff --git a/src/Common/FieldVisitorHash.h b/src/Common/FieldVisitorHash.h index 6c786fda4ad..bf7c3d5004f 100644 --- a/src/Common/FieldVisitorHash.h +++ b/src/Common/FieldVisitorHash.h @@ -16,6 +16,8 @@ public: FieldVisitorHash(SipHash & hash_); void operator() (const Null & x) const; + void operator() (const NegativeInfinity & x) const; + void operator() (const PositiveInfinity & x) const; void operator() (const UInt64 & x) const; void operator() (const UInt128 & x) const; void operator() (const UInt256 & x) const; diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp index 0064830c08a..e0ffca28341 100644 --- a/src/Common/FieldVisitorSum.cpp +++ b/src/Common/FieldVisitorSum.cpp @@ -22,6 +22,8 @@ bool FieldVisitorSum::operator() (UInt64 & x) const bool FieldVisitorSum::operator() (Float64 & x) const { x += get(rhs); return x != 0; } bool FieldVisitorSum::operator() (Null &) const { throw Exception("Cannot sum Nulls", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (NegativeInfinity &) const { throw Exception("Cannot sum -Inf", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (PositiveInfinity &) const { throw Exception("Cannot sum +Inf", ErrorCodes::LOGICAL_ERROR); } bool FieldVisitorSum::operator() (String &) const { throw Exception("Cannot sum Strings", ErrorCodes::LOGICAL_ERROR); } bool FieldVisitorSum::operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); } bool FieldVisitorSum::operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h index e208933043b..4c34fa86455 100644 --- a/src/Common/FieldVisitorSum.h +++ b/src/Common/FieldVisitorSum.h @@ -21,6 +21,8 @@ public: bool operator() (UInt64 & x) const; bool operator() (Float64 & x) const; bool operator() (Null &) const; + bool operator() (NegativeInfinity & x) const; + bool operator() (PositiveInfinity & x) const; bool operator() (String &) const; bool operator() (Array &) const; bool operator() (Tuple &) const; diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp index 45bc54f2c2a..74dfc55e1db 100644 --- a/src/Common/FieldVisitorToString.cpp +++ b/src/Common/FieldVisitorToString.cpp @@ -53,6 +53,8 @@ static String formatFloat(const Float64 x) String FieldVisitorToString::operator() (const Null &) const { return "NULL"; } +String FieldVisitorToString::operator() (const NegativeInfinity &) const { return "-Inf"; } +String FieldVisitorToString::operator() (const PositiveInfinity &) const { return "+Inf"; } String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); } String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); } String FieldVisitorToString::operator() (const Float64 & x) const { return formatFloat(x); } diff --git a/src/Common/FieldVisitorToString.h b/src/Common/FieldVisitorToString.h index 39709f1c272..139f011927f 100644 --- a/src/Common/FieldVisitorToString.h +++ b/src/Common/FieldVisitorToString.h @@ -10,6 +10,8 @@ class FieldVisitorToString : public StaticVisitor { public: String operator() (const Null & x) const; + String operator() (const NegativeInfinity & x) const; + String operator() (const PositiveInfinity & x) const; String operator() (const UInt64 & x) const; String operator() (const UInt128 & x) const; String operator() (const UInt256 & x) const; diff --git a/src/Common/FieldVisitorWriteBinary.cpp b/src/Common/FieldVisitorWriteBinary.cpp index 8e991ad13d3..56df9f1e43a 100644 --- a/src/Common/FieldVisitorWriteBinary.cpp +++ b/src/Common/FieldVisitorWriteBinary.cpp @@ -7,6 +7,8 @@ namespace DB { void FieldVisitorWriteBinary::operator() (const Null &, WriteBuffer &) const { } +void FieldVisitorWriteBinary::operator() (const NegativeInfinity &, WriteBuffer &) const { } +void FieldVisitorWriteBinary::operator() (const PositiveInfinity &, WriteBuffer &) const { } void FieldVisitorWriteBinary::operator() (const UInt64 & x, WriteBuffer & buf) const { writeVarUInt(x, buf); } void FieldVisitorWriteBinary::operator() (const Int64 & x, WriteBuffer & buf) const { writeVarInt(x, buf); } void FieldVisitorWriteBinary::operator() (const Float64 & x, WriteBuffer & buf) const { writeFloatBinary(x, buf); } diff --git a/src/Common/FieldVisitorWriteBinary.h b/src/Common/FieldVisitorWriteBinary.h index ae864ca74f3..5f7bf578e32 100644 --- a/src/Common/FieldVisitorWriteBinary.h +++ b/src/Common/FieldVisitorWriteBinary.h @@ -9,6 +9,8 @@ class FieldVisitorWriteBinary { public: void operator() (const Null & x, WriteBuffer & buf) const; + void operator() (const NegativeInfinity & x, WriteBuffer & buf) const; + void operator() (const PositiveInfinity & x, WriteBuffer & buf) const; void operator() (const UInt64 & x, WriteBuffer & buf) const; void operator() (const UInt128 & x, WriteBuffer & buf) const; void operator() (const UInt256 & x, WriteBuffer & buf) const; diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h index 0f605b7da23..55dedf8cef2 100644 --- a/src/Common/FieldVisitorsAccurateComparison.h +++ b/src/Common/FieldVisitorsAccurateComparison.h @@ -26,8 +26,12 @@ public: template bool operator() (const T & l, const U & r) const { - if constexpr (std::is_same_v || std::is_same_v) + if constexpr (std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v) + { return std::is_same_v; + } else { if constexpr (std::is_same_v) @@ -77,6 +81,10 @@ public: { if constexpr (std::is_same_v || std::is_same_v) return false; + else if constexpr (std::is_same_v || std::is_same_v) + return !std::is_same_v; + else if constexpr (std::is_same_v || std::is_same_v) + return false; else { if constexpr (std::is_same_v) diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index e625c92f826..b7b03951ac9 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -455,6 +455,16 @@ inline void writeText(const Null &, WriteBuffer & buf) writeText(std::string("NULL"), buf); } +inline void writeText(const NegativeInfinity &, WriteBuffer & buf) +{ + writeText(std::string("-Inf"), buf); +} + +inline void writeText(const PositiveInfinity &, WriteBuffer & buf) +{ + writeText(std::string("+Inf"), buf); +} + String toString(const Field & x) { return Field::dispatch( diff --git a/src/Core/Field.h b/src/Core/Field.h index 23569f5f9f1..ca6525c063b 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -218,6 +218,8 @@ template <> struct NearestFieldTypeImpl { using Type = Tuple; }; template <> struct NearestFieldTypeImpl { using Type = Map; }; template <> struct NearestFieldTypeImpl { using Type = UInt64; }; template <> struct NearestFieldTypeImpl { using Type = Null; }; +template <> struct NearestFieldTypeImpl { using Type = NegativeInfinity; }; +template <> struct NearestFieldTypeImpl { using Type = PositiveInfinity; }; template <> struct NearestFieldTypeImpl { using Type = AggregateFunctionStateData; }; @@ -251,11 +253,13 @@ public: enum Which { Null = 0, - UInt64 = 1, - Int64 = 2, - Float64 = 3, - UInt128 = 4, - Int128 = 5, + NegativeInfinity = 1, + PositiveInfinity = 2, + UInt64 = 3, + Int64 = 4, + Float64 = 5, + UInt128 = 6, + Int128 = 7, String = 16, Array = 17, @@ -276,6 +280,8 @@ public: switch (which) { case Null: return "Null"; + case NegativeInfinity: return "-Inf"; + case PositiveInfinity: return "+Inf"; case UInt64: return "UInt64"; case UInt128: return "UInt128"; case UInt256: return "UInt256"; @@ -404,7 +410,10 @@ public: Types::Which getType() const { return which; } const char * getTypeName() const { return Types::toString(which); } - bool isNull() const { return which == Types::Null; } + // Non-valued field are all denoted as Null + bool isNull() const { return which == Types::Null || which == Types::NegativeInfinity || which == Types::PositiveInfinity; } + bool isNegativeInfinity() const { return which == Types::NegativeInfinity; } + bool isPositiveInfinity() const { return which == Types::PositiveInfinity; } template @@ -459,7 +468,10 @@ public: switch (which) { - case Types::Null: return false; + case Types::Null: + case Types::NegativeInfinity: + case Types::PositiveInfinity: + return false; case Types::UInt64: return get() < rhs.get(); case Types::UInt128: return get() < rhs.get(); case Types::UInt256: return get() < rhs.get(); @@ -496,7 +508,10 @@ public: switch (which) { - case Types::Null: return true; + case Types::Null: + case Types::NegativeInfinity: + case Types::PositiveInfinity: + return true; case Types::UInt64: return get() <= rhs.get(); case Types::UInt128: return get() <= rhs.get(); case Types::UInt256: return get() <= rhs.get(); @@ -533,8 +548,11 @@ public: switch (which) { - case Types::Null: return true; - case Types::UInt64: return get() == rhs.get(); + case Types::Null: + case Types::NegativeInfinity: + case Types::PositiveInfinity: + return true; + case Types::UInt64: return get() == rhs.get(); case Types::Int64: return get() == rhs.get(); case Types::Float64: { @@ -573,6 +591,8 @@ public: switch (field.which) { case Types::Null: return f(field.template get()); + case Types::NegativeInfinity: return f(field.template get()); + case Types::PositiveInfinity: return f(field.template get()); // gcc 8.2.1 #if !defined(__clang__) #pragma GCC diagnostic push @@ -731,6 +751,8 @@ using Row = std::vector; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Null; }; +template <> struct Field::TypeToEnum { static const Types::Which value = Types::NegativeInfinity; }; +template <> struct Field::TypeToEnum { static const Types::Which value = Types::PositiveInfinity; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt64; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt128; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt256; }; @@ -751,6 +773,8 @@ template <> struct Field::TypeToEnum>{ static const Typ template <> struct Field::TypeToEnum{ static const Types::Which value = Types::AggregateFunctionState; }; template <> struct Field::EnumToType { using Type = Null; }; +template <> struct Field::EnumToType { using Type = NegativeInfinity; }; +template <> struct Field::EnumToType { using Type = PositiveInfinity; }; template <> struct Field::EnumToType { using Type = UInt64; }; template <> struct Field::EnumToType { using Type = UInt128; }; template <> struct Field::EnumToType { using Type = UInt256; }; diff --git a/src/Core/Types.h b/src/Core/Types.h index 151d9bdcf5a..158cb8b618c 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -14,6 +14,8 @@ namespace DB /// Data types for representing elementary values from a database in RAM. struct Null {}; +struct NegativeInfinity {}; +struct PositiveInfinity {}; /// Ignore strange gcc warning https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55776 #if !defined(__clang__) diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index c1a8cacd5c2..3c3439593ed 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -19,6 +19,7 @@ namespace DB namespace ErrorCodes { extern const int EMPTY_DATA_PASSED; + extern const int LOGICAL_ERROR; } @@ -27,6 +28,16 @@ DataTypePtr FieldToDataType::operator() (const Null &) const return std::make_shared(std::make_shared()); } +DataTypePtr FieldToDataType::operator() (const NegativeInfinity &) const +{ + throw Exception("It's invalid to have -inf literals in SQL", ErrorCodes::LOGICAL_ERROR); +} + +DataTypePtr FieldToDataType::operator() (const PositiveInfinity &) const +{ + throw Exception("It's invalid to have +inf literals in SQL", ErrorCodes::LOGICAL_ERROR); +} + DataTypePtr FieldToDataType::operator() (const UInt64 & x) const { if (x <= std::numeric_limits::max()) return std::make_shared(); diff --git a/src/DataTypes/FieldToDataType.h b/src/DataTypes/FieldToDataType.h index ca83ce868fc..6d579b2bf65 100644 --- a/src/DataTypes/FieldToDataType.h +++ b/src/DataTypes/FieldToDataType.h @@ -21,6 +21,8 @@ class FieldToDataType : public StaticVisitor { public: DataTypePtr operator() (const Null & x) const; + DataTypePtr operator() (const NegativeInfinity & x) const; + DataTypePtr operator() (const PositiveInfinity & x) const; DataTypePtr operator() (const UInt64 & x) const; DataTypePtr operator() (const UInt128 & x) const; DataTypePtr operator() (const UInt256 & x) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index fe52b30da7b..b625dd29b85 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -359,7 +359,7 @@ SetPtr ExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_o } -/// Performance optimisation for IN() if storage supports it. +/// Performance optimization for IN() if storage supports it. void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) { if (!node || !storage() || !storage()->supportsIndexForIn()) diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 66ba1f9ac9c..6ba8f44a82e 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -217,6 +217,8 @@ bool Set::insertFromBlock(const Block & block) set_elements[i] = filtered_column; else set_elements[i]->insertRangeFrom(*filtered_column, 0, filtered_column->size()); + if (transform_null_in && null_map_holder) + set_elements[i]->insert(Null{}); } } @@ -281,7 +283,7 @@ ColumnPtr Set::execute(const Block & block, bool negative) const key_columns.emplace_back() = materialized_columns.back().get(); } - /// We will check existence in Set only for keys, where all components are not NULL. + /// We will check existence in Set only for keys whose components do not contain any NULL value. ConstNullMapPtr null_map{}; ColumnPtr null_map_holder; if (!transform_null_in) @@ -408,7 +410,7 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector & key_ranges, { size_t tuple_size = indexes_mapping.size(); - ColumnsWithInfinity left_point; - ColumnsWithInfinity right_point; + FieldValues left_point; + FieldValues right_point; left_point.reserve(tuple_size); right_point.reserve(tuple_size); @@ -458,8 +460,8 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, right_point.emplace_back(ordered_set[i]->cloneEmpty()); } - bool invert_left_infinities = false; - bool invert_right_infinities = false; + bool left_included = true; + bool right_included = true; for (size_t i = 0; i < tuple_size; ++i) { @@ -471,48 +473,29 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, if (!new_range) return {true, true}; - /** A range that ends in (x, y, ..., +inf) exclusive is the same as a range - * that ends in (x, y, ..., -inf) inclusive and vice versa for the left bound. - */ - if (new_range->left_bounded) - { - if (!new_range->left_included) - invert_left_infinities = true; - - left_point[i].update(new_range->left); - } - else - { - if (invert_left_infinities) - left_point[i].update(ValueWithInfinity::PLUS_INFINITY); - else - left_point[i].update(ValueWithInfinity::MINUS_INFINITY); - } - - if (new_range->right_bounded) - { - if (!new_range->right_included) - invert_right_infinities = true; - - right_point[i].update(new_range->right); - } - else - { - if (invert_right_infinities) - right_point[i].update(ValueWithInfinity::MINUS_INFINITY); - else - right_point[i].update(ValueWithInfinity::PLUS_INFINITY); - } + left_point[i].update(new_range->left); + left_included &= new_range->left_included; + right_point[i].update(new_range->right); + right_included &= new_range->right_included; } - auto compare = [](const IColumn & lhs, const ValueWithInfinity & rhs, size_t row) + /// lhs < rhs return -1 + /// lhs == rhs return 0 + /// lhs > rhs return 1 + auto compare = [](const IColumn & lhs, const FieldValue & rhs, size_t row) { - auto type = rhs.getType(); - /// Return inverted infinity sign, because in 'lhs' all values are finite. - if (type != ValueWithInfinity::NORMAL) - return -static_cast(type); - - return lhs.compareAt(row, 0, rhs.getColumnIfFinite(), 1); + if (rhs.isNegativeInfinity()) + return 1; + if (rhs.isPositiveInfinity()) + { + Field f; + lhs.get(row, f); + if (f.isNull()) + return 0; // +Inf == +Inf + else + return -1; + } + return lhs.compareAt(row, 0, *rhs.column, 1); }; auto less = [this, &compare, tuple_size](size_t row, const auto & point) @@ -535,29 +518,27 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, }; /** Because each hyperrectangle maps to a contiguous sequence of elements - * laid out in the lexicographically increasing order, the set intersects the range - * if and only if either bound coincides with an element or at least one element - * is between the lower bounds - */ + * laid out in the lexicographically increasing order, the set intersects the range + * if and only if either bound coincides with an element or at least one element + * is between the lower bounds + */ auto indices = collections::range(0, size()); auto left_lower = std::lower_bound(indices.begin(), indices.end(), left_point, less); auto right_lower = std::lower_bound(indices.begin(), indices.end(), right_point, less); - /// A special case of 1-element KeyRange. It's useful for partition pruning + /// A special case of 1-element KeyRange. It's useful for partition pruning. + /// We don't take NULL into account here, as it's special in IN operator. bool one_element_range = true; for (size_t i = 0; i < tuple_size; ++i) { auto & left = left_point[i]; auto & right = right_point[i]; - if (left.getType() == right.getType()) + if (left.isNormal() && right.isNormal()) { - if (left.getType() == ValueWithInfinity::NORMAL) + if (0 != left.column->compareAt(0, 0, *right.column, 1)) { - if (0 != left.getColumnIfFinite().compareAt(0, 0, right.getColumnIfFinite(), 1)) - { - one_element_range = false; - break; - } + one_element_range = false; + break; } } else @@ -577,13 +558,46 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, return {false, true}; } - return - { - left_lower != right_lower - || (left_lower != indices.end() && equals(*left_lower, left_point)) - || (right_lower != indices.end() && equals(*right_lower, right_point)), - true - }; + /// If there are more than one element in the range, it can always be false. Thus we only need to check if it may be true or not. + + /// Given X >= x, Y >= y, find if there may be an element z in [X, Y]. Note + /// Case 1: [x ... y] + if (left_included && right_included) + return {left_lower != right_lower || (right_lower != indices.end() && equals(*right_lower, right_point)), true}; + /// Case 2: [x ... y) + else if (left_included && !right_included) + // X, Y + // x, y + return {left_lower != right_lower, true}; + /// Case 3: (x ... y] + else if (!left_included && right_included) + return + { + // X, M, ..., Y + // x, y + left_lower + 1 < right_lower + // X, Y, ... + // x, y + || (right_lower != indices.end() && equals(*right_lower, right_point)) + // X, Y + // x, y + || (left_lower < right_lower && !equals(*left_lower, left_point)), + + true + }; + /// Case 4: (x ... y) + else + return + { + // X, M, ..., Y + // x, y + left_lower + 1 < right_lower + // X, Y + // x, y + || (left_lower < right_lower && !equals(*left_lower, left_point)), + + true + }; } bool MergeTreeSetIndex::hasMonotonicFunctionsChain() const @@ -594,23 +608,18 @@ bool MergeTreeSetIndex::hasMonotonicFunctionsChain() const return false; } -void ValueWithInfinity::update(const Field & x) +void FieldValue::update(const Field & x) { - /// Keep at most one element in column. - if (!column->empty()) - column->popBack(1); - column->insert(x); - type = NORMAL; -} - -const IColumn & ValueWithInfinity::getColumnIfFinite() const -{ -#ifndef NDEBUG - if (type != NORMAL) - throw Exception("Trying to get column of infinite type", ErrorCodes::LOGICAL_ERROR); -#endif - - return *column; + if (x.isNegativeInfinity() || x.isPositiveInfinity()) + value = x; + else + { + /// Keep at most one element in column. + if (!column->empty()) + column->popBack(1); + column->insert(x); + value = Field(); // Set back to normal value. + } } } diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index c9bfbf0625c..9bf6630b844 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -178,29 +178,19 @@ using FunctionPtr = std::shared_ptr; * Single field is stored in column for more optimal inplace comparisons with other regular columns. * Extracting fields from columns and further their comparison is suboptimal and requires extra copying. */ -class ValueWithInfinity +struct FieldValue { -public: - enum Type - { - MINUS_INFINITY = -1, - NORMAL = 0, - PLUS_INFINITY = 1 - }; - - ValueWithInfinity(MutableColumnPtr && column_) - : column(std::move(column_)), type(NORMAL) {} - + FieldValue(MutableColumnPtr && column_) : column(std::move(column_)) {} void update(const Field & x); - void update(Type type_) { type = type_; } - const IColumn & getColumnIfFinite() const; + bool isNormal() const { return !value.isPositiveInfinity() && !value.isNegativeInfinity(); } + bool isPositiveInfinity() const { return value.isPositiveInfinity(); } + bool isNegativeInfinity() const { return value.isNegativeInfinity(); } - Type getType() const { return type; } + Field value; // Null, -Inf, +Inf -private: + // If value is Null, uses the actual value in column MutableColumnPtr column; - Type type; }; @@ -230,7 +220,7 @@ private: Columns ordered_set; std::vector indexes_mapping; - using ColumnsWithInfinity = std::vector; + using FieldValues = std::vector; }; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8fe6a0a484b..4ae6c7215be 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -78,6 +79,12 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis Field max_val; serialization->deserializeBinary(max_val, *file); + // NULL_LAST + if (min_val.isNull()) + min_val = PositiveInfinity(); + if (max_val.isNull()) + max_val = PositiveInfinity(); + hyperrectangle.emplace_back(min_val, true, max_val, true); } initialized = true; @@ -132,14 +139,19 @@ void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & FieldRef min_value; FieldRef max_value; const ColumnWithTypeAndName & column = block.getByName(column_names[i]); - column.column->getExtremes(min_value, max_value); + if (const auto * column_nullable = typeid_cast(column.column.get())) + column_nullable->getExtremesNullLast(min_value, max_value); + else + column.column->getExtremes(min_value, max_value); if (!initialized) hyperrectangle.emplace_back(min_value, true, max_value, true); else { - hyperrectangle[i].left = std::min(hyperrectangle[i].left, min_value); - hyperrectangle[i].right = std::max(hyperrectangle[i].right, max_value); + hyperrectangle[i].left + = applyVisitor(FieldVisitorAccurateLess(), hyperrectangle[i].left, min_value) ? hyperrectangle[i].left : min_value; + hyperrectangle[i].right + = applyVisitor(FieldVisitorAccurateLess(), hyperrectangle[i].right, max_value) ? max_value : hyperrectangle[i].right; } } diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 246d00b831b..c1ded19f7fe 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -43,15 +43,8 @@ String Range::toString() const { WriteBufferFromOwnString str; - if (!left_bounded) - str << "(-inf, "; - else - str << (left_included ? '[' : '(') << applyVisitor(FieldVisitorToString(), left) << ", "; - - if (!right_bounded) - str << "+inf)"; - else - str << applyVisitor(FieldVisitorToString(), right) << (right_included ? ']' : ')'); + str << (left_included ? '[' : '(') << applyVisitor(FieldVisitorToString(), left) << ", "; + str << applyVisitor(FieldVisitorToString(), right) << (right_included ? ']' : ')'); return str.str(); } @@ -205,6 +198,38 @@ const KeyCondition::AtomMap KeyCondition::atom_map return true; } }, + { + "nullIn", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_IN_SET; + return true; + } + }, + { + "notNullIn", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_NOT_IN_SET; + return true; + } + }, + { + "globalNullIn", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_IN_SET; + return true; + } + }, + { + "globalNotNullIn", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_NOT_IN_SET; + return true; + } + }, { "empty", [] (RPNElement & out, const Field & value) @@ -291,6 +316,26 @@ const KeyCondition::AtomMap KeyCondition::atom_map return true; } + }, + { + "isNotNull", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_IS_NOT_NULL; + // isNotNull means (-Inf, +Inf), which is the default Range + out.range = Range(); + return true; + } + }, + { + "isNull", + [] (RPNElement & out, const Field &) + { + out.function = RPNElement::FUNCTION_IS_NULL; + // When using NULL_LAST, isNull means [+Inf, +Inf] + out.range = Range(Field(PositiveInfinity{})); + return true; + } } }; @@ -304,6 +349,14 @@ static const std::map inverse_relations = { {"lessOrEquals", "greater"}, {"in", "notIn"}, {"notIn", "in"}, + {"globalIn", "globalNotIn"}, + {"globalNotIn", "globalIn"}, + {"nullIn", "notNullIn"}, + {"notNullIn", "nullIn"}, + {"globalNullIn", "globalNotNullIn"}, + {"globalNullNotIn", "globalNullIn"}, + {"isNull", "isNotNull"}, + {"isNotNull", "isNull"}, {"like", "notLike"}, {"notLike", "like"}, {"empty", "notEmpty"}, @@ -613,7 +666,6 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( if (key_subexpr_names.count(expr_name) == 0) return false; - /// TODO Nullable index is not yet landed. if (out_value.isNull()) return false; @@ -735,7 +787,6 @@ bool KeyCondition::canConstantBeWrappedByFunctions( const auto & sample_block = key_expr->getSampleBlock(); - /// TODO Nullable index is not yet landed. if (out_value.isNull()) return false; @@ -1134,7 +1185,7 @@ static void castValueToType(const DataTypePtr & desired_type, Field & src_value, bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context, Block & block_with_constants, RPNElement & out) { - /** Functions < > = != <= >= in `notIn`, where one argument is a constant, and the other is one of columns of key, + /** Functions < > = != <= >= in `notIn` isNull isNotNull, where one argument is a constant, and the other is one of columns of key, * or itself, wrapped in a chain of possibly-monotonic functions, * or constant expression - number. */ @@ -1179,8 +1230,8 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context, /// If we use this key condition to prune partitions by single value, we cannot relax conditions for NOT. if (single_point - && (func_name == "notLike" || func_name == "notIn" || func_name == "globalNotIn" || func_name == "notEquals" - || func_name == "notEmpty")) + && (func_name == "notLike" || func_name == "notIn" || func_name == "globalNotIn" || func_name == "notNullIn" + || func_name == "globalNotNullIn" || func_name == "notEquals" || func_name == "notEmpty")) strict_condition = true; if (functionIsInOrGlobalInOperator(func_name)) @@ -1491,6 +1542,8 @@ KeyCondition::Description KeyCondition::getDescription() const else if ( element.function == RPNElement::FUNCTION_IN_RANGE || element.function == RPNElement::FUNCTION_NOT_IN_RANGE + || element.function == RPNElement::FUNCTION_IS_NULL + || element.function == RPNElement::FUNCTION_IS_NOT_NULL || element.function == RPNElement::FUNCTION_IN_SET || element.function == RPNElement::FUNCTION_NOT_IN_SET) { @@ -1655,11 +1708,13 @@ KeyCondition::Description KeyCondition::getDescription() const * over at least one hyperrectangle from which this range consists. */ +FieldRef negativeInfinity(NegativeInfinity{}), positiveInfinity(PositiveInfinity{}); + template static BoolMask forAnyHyperrectangle( size_t key_size, - const FieldRef * key_left, - const FieldRef * key_right, + const FieldRef * left_keys, + const FieldRef * right_keys, bool left_bounded, bool right_bounded, std::vector & hyperrectangle, @@ -1675,10 +1730,10 @@ static BoolMask forAnyHyperrectangle( /// Let's go through the matching elements of the key. while (prefix_size < key_size) { - if (key_left[prefix_size] == key_right[prefix_size]) + if (left_keys[prefix_size] == right_keys[prefix_size]) { /// Point ranges. - hyperrectangle[prefix_size] = Range(key_left[prefix_size]); + hyperrectangle[prefix_size] = Range(left_keys[prefix_size]); ++prefix_size; } else @@ -1692,11 +1747,11 @@ static BoolMask forAnyHyperrectangle( if (prefix_size + 1 == key_size) { if (left_bounded && right_bounded) - hyperrectangle[prefix_size] = Range(key_left[prefix_size], true, key_right[prefix_size], true); + hyperrectangle[prefix_size] = Range(left_keys[prefix_size], true, right_keys[prefix_size], true); else if (left_bounded) - hyperrectangle[prefix_size] = Range::createLeftBounded(key_left[prefix_size], true); + hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], true); else if (right_bounded) - hyperrectangle[prefix_size] = Range::createRightBounded(key_right[prefix_size], true); + hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], true); return callback(hyperrectangle); } @@ -1704,11 +1759,11 @@ static BoolMask forAnyHyperrectangle( /// (x1 .. x2) x (-inf .. +inf) if (left_bounded && right_bounded) - hyperrectangle[prefix_size] = Range(key_left[prefix_size], false, key_right[prefix_size], false); + hyperrectangle[prefix_size] = Range(left_keys[prefix_size], false, right_keys[prefix_size], false); else if (left_bounded) - hyperrectangle[prefix_size] = Range::createLeftBounded(key_left[prefix_size], false); + hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], false); else if (right_bounded) - hyperrectangle[prefix_size] = Range::createRightBounded(key_right[prefix_size], false); + hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], false); for (size_t i = prefix_size + 1; i < key_size; ++i) hyperrectangle[i] = Range(); @@ -1728,8 +1783,8 @@ static BoolMask forAnyHyperrectangle( if (left_bounded) { - hyperrectangle[prefix_size] = Range(key_left[prefix_size]); - result = result | forAnyHyperrectangle(key_size, key_left, key_right, true, false, hyperrectangle, prefix_size + 1, initial_mask, callback); + hyperrectangle[prefix_size] = Range(left_keys[prefix_size]); + result = result | forAnyHyperrectangle(key_size, left_keys, right_keys, true, false, hyperrectangle, prefix_size + 1, initial_mask, callback); if (result.isComplete()) return result; } @@ -1738,8 +1793,8 @@ static BoolMask forAnyHyperrectangle( if (right_bounded) { - hyperrectangle[prefix_size] = Range(key_right[prefix_size]); - result = result | forAnyHyperrectangle(key_size, key_left, key_right, false, true, hyperrectangle, prefix_size + 1, initial_mask, callback); + hyperrectangle[prefix_size] = Range(right_keys[prefix_size]); + result = result | forAnyHyperrectangle(key_size, left_keys, right_keys, false, true, hyperrectangle, prefix_size + 1, initial_mask, callback); if (result.isComplete()) return result; } @@ -1750,37 +1805,31 @@ static BoolMask forAnyHyperrectangle( BoolMask KeyCondition::checkInRange( size_t used_key_size, - const FieldRef * left_key, - const FieldRef * right_key, + const FieldRef * left_keys, + const FieldRef * right_keys, const DataTypes & data_types, - bool right_bounded, BoolMask initial_mask) const { std::vector key_ranges(used_key_size, Range()); -/* std::cerr << "Checking for: ["; - for (size_t i = 0; i != used_key_size; ++i) - std::cerr << (i != 0 ? ", " : "") << applyVisitor(FieldVisitorToString(), left_key[i]); - std::cerr << " ... "; + // std::cerr << "Checking for: ["; + // for (size_t i = 0; i != used_key_size; ++i) + // std::cerr << (i != 0 ? ", " : "") << applyVisitor(FieldVisitorToString(), left_keys[i]); + // std::cerr << " ... "; - if (right_bounded) - { - for (size_t i = 0; i != used_key_size; ++i) - std::cerr << (i != 0 ? ", " : "") << applyVisitor(FieldVisitorToString(), right_key[i]); - std::cerr << "]\n"; - } - else - std::cerr << "+inf)\n";*/ + // for (size_t i = 0; i != used_key_size; ++i) + // std::cerr << (i != 0 ? ", " : "") << applyVisitor(FieldVisitorToString(), right_keys[i]); + // std::cerr << "]\n"; - return forAnyHyperrectangle(used_key_size, left_key, right_key, true, right_bounded, key_ranges, 0, initial_mask, + return forAnyHyperrectangle(used_key_size, left_keys, right_keys, true, true, key_ranges, 0, initial_mask, [&] (const std::vector & key_ranges_hyperrectangle) { auto res = checkInHyperrectangle(key_ranges_hyperrectangle, data_types); -/* std::cerr << "Hyperrectangle: "; - for (size_t i = 0, size = key_ranges.size(); i != size; ++i) - std::cerr << (i != 0 ? " x " : "") << key_ranges[i].toString(); - std::cerr << ": " << res.can_be_true << "\n";*/ + // std::cerr << "Hyperrectangle: "; + // for (size_t i = 0, size = key_ranges.size(); i != size; ++i) + // std::cerr << (i != 0 ? " x " : "") << key_ranges[i].toString(); + // std::cerr << ": " << res.can_be_true << "\n"; return res; }); @@ -1808,6 +1857,8 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( /// If we apply function to open interval, we can get empty intervals in result. /// E.g. for ('2020-01-03', '2020-01-20') after applying 'toYYYYMM' we will get ('202001', '202001'). /// To avoid this we make range left and right included. + /// Any function that treats NULL specially is not monotonic. + /// Thus we can safely use isNull() as an -Inf/+Inf indicator here. if (!key_range.left.isNull()) { key_range.left = applyFunction(func, current_type, key_range.left); @@ -1823,7 +1874,7 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( current_type = func->getResultType(); if (!monotonicity.is_positive) - key_range.swapLeftAndRight(); + key_range.invert(); } return key_range; } @@ -1948,6 +1999,17 @@ BoolMask KeyCondition::checkInHyperrectangle( if (element.function == RPNElement::FUNCTION_NOT_IN_RANGE) rpn_stack.back() = !rpn_stack.back(); } + else if ( + element.function == RPNElement::FUNCTION_IS_NULL + || element.function == RPNElement::FUNCTION_IS_NOT_NULL) + { + const Range * key_range = &hyperrectangle[element.key_column]; + + /// No need to apply monotonic functions as nulls are kept. + bool intersects = element.range.intersectsRange(*key_range); + bool contains = element.range.containsRange(*key_range); + rpn_stack.emplace_back(intersects, !contains); + } else if ( element.function == RPNElement::FUNCTION_IN_SET || element.function == RPNElement::FUNCTION_NOT_IN_SET) @@ -2002,43 +2064,13 @@ BoolMask KeyCondition::checkInHyperrectangle( } -BoolMask KeyCondition::checkInRange( - size_t used_key_size, - const FieldRef * left_key, - const FieldRef * right_key, - const DataTypes & data_types, - BoolMask initial_mask) const -{ - return checkInRange(used_key_size, left_key, right_key, data_types, true, initial_mask); -} - - bool KeyCondition::mayBeTrueInRange( size_t used_key_size, - const FieldRef * left_key, - const FieldRef * right_key, + const FieldRef * left_keys, + const FieldRef * right_keys, const DataTypes & data_types) const { - return checkInRange(used_key_size, left_key, right_key, data_types, true, BoolMask::consider_only_can_be_true).can_be_true; -} - - -BoolMask KeyCondition::checkAfter( - size_t used_key_size, - const FieldRef * left_key, - const DataTypes & data_types, - BoolMask initial_mask) const -{ - return checkInRange(used_key_size, left_key, nullptr, data_types, false, initial_mask); -} - - -bool KeyCondition::mayBeTrueAfter( - size_t used_key_size, - const FieldRef * left_key, - const DataTypes & data_types) const -{ - return checkInRange(used_key_size, left_key, nullptr, data_types, false, BoolMask::consider_only_can_be_true).can_be_true; + return checkInRange(used_key_size, left_keys, right_keys, data_types, BoolMask::consider_only_can_be_true).can_be_true; } String KeyCondition::RPNElement::toString() const { return toString("column " + std::to_string(key_column), false); } @@ -2108,6 +2140,15 @@ String KeyCondition::RPNElement::toString(const std::string_view & column_name, buf << ")"; return buf.str(); } + case FUNCTION_IS_NULL: + case FUNCTION_IS_NOT_NULL: + { + buf << "("; + print_wrapped_column(buf); + buf << (function == FUNCTION_IS_NULL ? " isNull" : " isNotNull"); + buf << ")"; + return buf.str(); + } case ALWAYS_FALSE: return "false"; case ALWAYS_TRUE: @@ -2149,6 +2190,8 @@ bool KeyCondition::unknownOrAlwaysTrue(bool unknown_any) const || element.function == RPNElement::FUNCTION_IN_RANGE || element.function == RPNElement::FUNCTION_IN_SET || element.function == RPNElement::FUNCTION_NOT_IN_SET + || element.function == RPNElement::FUNCTION_IS_NULL + || element.function == RPNElement::FUNCTION_IS_NOT_NULL || element.function == RPNElement::ALWAYS_FALSE) { rpn_stack.push_back(false); @@ -2192,6 +2235,8 @@ size_t KeyCondition::getMaxKeyColumn() const { if (element.function == RPNElement::FUNCTION_NOT_IN_RANGE || element.function == RPNElement::FUNCTION_IN_RANGE + || element.function == RPNElement::FUNCTION_IS_NULL + || element.function == RPNElement::FUNCTION_IS_NOT_NULL || element.function == RPNElement::FUNCTION_IN_SET || element.function == RPNElement::FUNCTION_NOT_IN_SET) { diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 7e7b767b53b..3754457191e 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -55,25 +55,24 @@ private: static bool less(const Field & lhs, const Field & rhs); public: - FieldRef left; /// the left border, if any - FieldRef right; /// the right border, if any - bool left_bounded = false; /// bounded at the left - bool right_bounded = false; /// bounded at the right - bool left_included = false; /// includes the left border, if any - bool right_included = false; /// includes the right border, if any + FieldRef left = NegativeInfinity{}; /// the left border + FieldRef right = PositiveInfinity{}; /// the right border + bool left_included = false; /// includes the left border + bool right_included = false; /// includes the right border - /// The whole unversum. + /// The whole universe (not null). Range() {} /// One point. Range(const FieldRef & point) - : left(point), right(point), left_bounded(true), right_bounded(true), left_included(true), right_included(true) {} + : left(point), right(point), left_included(true), right_included(true) {} /// A bounded two-sided range. Range(const FieldRef & left_, bool left_included_, const FieldRef & right_, bool right_included_) - : left(left_), right(right_), - left_bounded(true), right_bounded(true), - left_included(left_included_), right_included(right_included_) + : left(left_) + , right(right_) + , left_included(left_included_) + , right_included(right_included_) { shrinkToIncludedIfPossible(); } @@ -82,9 +81,11 @@ public: { Range r; r.right = right_point; - r.right_bounded = true; r.right_included = right_included; r.shrinkToIncludedIfPossible(); + // Special case for [-Inf, -Inf] + if (r.right.isNegativeInfinity() && right_included) + r.left_included = true; return r; } @@ -92,9 +93,11 @@ public: { Range r; r.left = left_point; - r.left_bounded = true; r.left_included = left_included; r.shrinkToIncludedIfPossible(); + // Special case for [+Inf, +Inf] + if (r.left.isPositiveInfinity() && left_included) + r.right_included = true; return r; } @@ -104,7 +107,7 @@ public: */ void shrinkToIncludedIfPossible() { - if (left.isExplicit() && left_bounded && !left_included) + if (left.isExplicit() && !left_included) { if (left.getType() == Field::Types::UInt64 && left.get() != std::numeric_limits::max()) { @@ -117,7 +120,7 @@ public: left_included = true; } } - if (right.isExplicit() && right_bounded && !right_included) + if (right.isExplicit() && !right_included) { if (right.getType() == Field::Types::UInt64 && right.get() != std::numeric_limits::min()) { @@ -132,12 +135,7 @@ public: } } - bool empty() const - { - return left_bounded && right_bounded - && (less(right, left) - || ((!left_included || !right_included) && !less(left, right))); - } + bool empty() const { return less(right, left) || ((!left_included || !right_included) && !less(left, right)); } /// x contained in the range bool contains(const FieldRef & x) const @@ -148,35 +146,23 @@ public: /// x is to the left bool rightThan(const FieldRef & x) const { - return (left_bounded - ? !(less(left, x) || (left_included && equals(x, left))) - : false); + return less(left, x) || (left_included && equals(x, left)); } /// x is to the right bool leftThan(const FieldRef & x) const { - return (right_bounded - ? !(less(x, right) || (right_included && equals(x, right))) - : false); + return less(x, right) || (right_included && equals(x, right)); } bool intersectsRange(const Range & r) const { /// r to the left of me. - if (r.right_bounded - && left_bounded - && (less(r.right, left) - || ((!left_included || !r.right_included) - && equals(r.right, left)))) + if (less(r.right, left) || ((!left_included || !r.right_included) && equals(r.right, left))) return false; /// r to the right of me. - if (r.left_bounded - && right_bounded - && (less(right, r.left) /// ...} {... - || ((!right_included || !r.left_included) /// ...) [... or ...] (... - && equals(r.left, right)))) + if (less(right, r.left) || ((!right_included || !r.left_included) && equals(r.left, right))) return false; return true; @@ -185,30 +171,23 @@ public: bool containsRange(const Range & r) const { /// r starts to the left of me. - if (left_bounded - && (!r.left_bounded - || less(r.left, left) - || (r.left_included - && !left_included - && equals(r.left, left)))) + if (less(r.left, left) || (r.left_included && !left_included && equals(r.left, left))) return false; /// r ends right of me. - if (right_bounded - && (!r.right_bounded - || less(right, r.right) - || (r.right_included - && !right_included - && equals(r.right, right)))) + if (less(right, r.right) || (r.right_included && !right_included && equals(r.right, right))) return false; return true; } - void swapLeftAndRight() + void invert() { std::swap(left, right); - std::swap(left_bounded, right_bounded); + if (left.isPositiveInfinity()) + left = NegativeInfinity{}; + if (right.isNegativeInfinity()) + right = PositiveInfinity{}; std::swap(left_included, right_included); } @@ -247,16 +226,8 @@ public: /// one of the resulting mask components (see BoolMask::consider_only_can_be_XXX). BoolMask checkInRange( size_t used_key_size, - const FieldRef * left_key, - const FieldRef* right_key, - const DataTypes & data_types, - BoolMask initial_mask = BoolMask(false, false)) const; - - /// Are the condition and its negation valid in a semi-infinite (not limited to the right) key range. - /// left_key must contain all the fields in the sort_descr in the appropriate order. - BoolMask checkAfter( - size_t used_key_size, - const FieldRef * left_key, + const FieldRef * left_keys, + const FieldRef * right_keys, const DataTypes & data_types, BoolMask initial_mask = BoolMask(false, false)) const; @@ -264,15 +235,8 @@ public: /// This is more efficient than checkInRange(...).can_be_true. bool mayBeTrueInRange( size_t used_key_size, - const FieldRef * left_key, - const FieldRef * right_key, - const DataTypes & data_types) const; - - /// Same as checkAfter, but calculate only may_be_true component of a result. - /// This is more efficient than checkAfter(...).can_be_true. - bool mayBeTrueAfter( - size_t used_key_size, - const FieldRef * left_key, + const FieldRef * left_keys, + const FieldRef * right_keys, const DataTypes & data_types) const; /// Checks that the index can not be used @@ -338,6 +302,8 @@ private: FUNCTION_NOT_IN_RANGE, FUNCTION_IN_SET, FUNCTION_NOT_IN_SET, + FUNCTION_IS_NULL, + FUNCTION_IS_NOT_NULL, FUNCTION_UNKNOWN, /// Can take any value. /// Operators of the logical expression. FUNCTION_NOT, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ae3b533918d..e293926995d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1294,6 +1294,9 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( create_field_ref = [index_columns](size_t row, size_t column, FieldRef & field) { field = {index_columns.get(), row, column}; + // NULL_LAST + if (field.isNull()) + field = PositiveInfinity{}; }; } else @@ -1301,6 +1304,9 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( create_field_ref = [&index](size_t row, size_t column, FieldRef & field) { index[column]->get(row, field); + // NULL_LAST + if (field.isNull()) + field = PositiveInfinity{}; }; } @@ -1313,21 +1319,22 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( if (range.end == marks_count && !has_final_mark) { for (size_t i = 0; i < used_key_size; ++i) + { create_field_ref(range.begin, i, index_left[i]); - - return key_condition.mayBeTrueAfter( - used_key_size, index_left.data(), primary_key.data_types); + index_right[i] = PositiveInfinity{}; + } } - - if (has_final_mark && range.end == marks_count) - range.end -= 1; /// Remove final empty mark. It's useful only for primary key condition. - - for (size_t i = 0; i < used_key_size; ++i) + else { - create_field_ref(range.begin, i, index_left[i]); - create_field_ref(range.end, i, index_right[i]); - } + if (has_final_mark && range.end == marks_count) + range.end -= 1; /// Remove final empty mark. It's useful only for primary key condition. + for (size_t i = 0; i < used_key_size; ++i) + { + create_field_ref(range.begin, i, index_left[i]); + create_field_ref(range.end, i, index_right[i]); + } + } return key_condition.mayBeTrueInRange( used_key_size, index_left.data(), index_right.data(), primary_key.data_types); }; diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index 099d561cf80..e29a6198252 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -5,6 +5,7 @@ #include #include +#include namespace DB { @@ -38,22 +39,8 @@ void MergeTreeIndexGranuleMinMax::serializeBinary(WriteBuffer & ostr) const { const DataTypePtr & type = index_sample_block.getByPosition(i).type; auto serialization = type->getDefaultSerialization(); - - if (!type->isNullable()) - { - serialization->serializeBinary(hyperrectangle[i].left, ostr); - serialization->serializeBinary(hyperrectangle[i].right, ostr); - } - else - { - bool is_null = hyperrectangle[i].left.isNull() || hyperrectangle[i].right.isNull(); // one is enough - writeBinary(is_null, ostr); - if (!is_null) - { - serialization->serializeBinary(hyperrectangle[i].left, ostr); - serialization->serializeBinary(hyperrectangle[i].right, ostr); - } - } + serialization->serializeBinary(hyperrectangle[i].left, ostr); + serialization->serializeBinary(hyperrectangle[i].right, ostr); } } @@ -63,32 +50,18 @@ void MergeTreeIndexGranuleMinMax::deserializeBinary(ReadBuffer & istr) Field min_val; Field max_val; - for (size_t i = 0; i < index_sample_block.columns(); ++i) { const DataTypePtr & type = index_sample_block.getByPosition(i).type; auto serialization = type->getDefaultSerialization(); + serialization->deserializeBinary(min_val, istr); + serialization->deserializeBinary(max_val, istr); - if (!type->isNullable()) - { - serialization->deserializeBinary(min_val, istr); - serialization->deserializeBinary(max_val, istr); - } - else - { - bool is_null; - readBinary(is_null, istr); - if (!is_null) - { - serialization->deserializeBinary(min_val, istr); - serialization->deserializeBinary(max_val, istr); - } - else - { - min_val = Null(); - max_val = Null(); - } - } + // NULL_LAST + if (min_val.isNull()) + min_val = PositiveInfinity(); + if (max_val.isNull()) + max_val = PositiveInfinity(); hyperrectangle.emplace_back(min_val, true, max_val, true); } } @@ -117,8 +90,11 @@ void MergeTreeIndexAggregatorMinMax::update(const Block & block, size_t * pos, s for (size_t i = 0; i < index_sample_block.columns(); ++i) { auto index_column_name = index_sample_block.getByPosition(i).name; - const auto & column = block.getByName(index_column_name).column; - column->cut(*pos, rows_read)->getExtremes(field_min, field_max); + const auto & column = block.getByName(index_column_name).column->cut(*pos, rows_read); + if (const auto * column_nullable = typeid_cast(column.get())) + column_nullable->getExtremesNullLast(field_min, field_max); + else + column->getExtremes(field_min, field_max); if (hyperrectangle.size() <= i) { @@ -126,8 +102,10 @@ void MergeTreeIndexAggregatorMinMax::update(const Block & block, size_t * pos, s } else { - hyperrectangle[i].left = std::min(hyperrectangle[i].left, field_min); - hyperrectangle[i].right = std::max(hyperrectangle[i].right, field_max); + hyperrectangle[i].left + = applyVisitor(FieldVisitorAccurateLess(), hyperrectangle[i].left, field_min) ? hyperrectangle[i].left : field_min; + hyperrectangle[i].right + = applyVisitor(FieldVisitorAccurateLess(), hyperrectangle[i].right, field_max) ? field_max : hyperrectangle[i].right; } } @@ -156,9 +134,6 @@ bool MergeTreeIndexConditionMinMax::mayBeTrueOnGranule(MergeTreeIndexGranulePtr if (!granule) throw Exception( "Minmax index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR); - for (const auto & range : granule->hyperrectangle) - if (range.left.isNull() || range.right.isNull()) - return true; return condition.checkInHyperrectangle(granule->hyperrectangle, index_data_types).can_be_true; } diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 8c027eb2089..0d457971dc6 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -43,6 +43,16 @@ namespace UInt8 type = Field::Types::Null; hash.update(type); } + void operator() (const NegativeInfinity &) const + { + UInt8 type = Field::Types::NegativeInfinity; + hash.update(type); + } + void operator() (const PositiveInfinity &) const + { + UInt8 type = Field::Types::PositiveInfinity; + hash.update(type); + } void operator() (const UInt64 & x) const { UInt8 type = Field::Types::UInt64; diff --git a/tests/queries/0_stateless/00944_minmax_null.sql b/tests/queries/0_stateless/00944_minmax_null.sql index 01b86775481..71ae2e77205 100644 --- a/tests/queries/0_stateless/00944_minmax_null.sql +++ b/tests/queries/0_stateless/00944_minmax_null.sql @@ -3,7 +3,7 @@ DROP TABLE IF EXISTS min_max_with_nullable_string; CREATE TABLE min_max_with_nullable_string ( t DateTime, nullable_str Nullable(String), - INDEX nullable_str_min_max nullable_str TYPE minmax GRANULARITY 8192 + INDEX nullable_str_min_max nullable_str TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree ORDER BY (t); INSERT INTO min_max_with_nullable_string(t) VALUES (now()) (now()); @@ -20,4 +20,4 @@ SELECT count() FROM min_max_with_nullable_string WHERE nullable_str = '.'; SELECT count() FROM min_max_with_nullable_string WHERE nullable_str = ''; -DROP TABLE min_max_with_nullable_string; \ No newline at end of file +DROP TABLE min_max_with_nullable_string; diff --git a/tests/queries/0_stateless/01410_nullable_key.reference b/tests/queries/0_stateless/01410_nullable_key.reference deleted file mode 100644 index 75163f1bf41..00000000000 --- a/tests/queries/0_stateless/01410_nullable_key.reference +++ /dev/null @@ -1,35 +0,0 @@ -0 0 -2 3 -4 6 -6 9 -8 12 -10 15 -12 18 -14 21 -16 24 -18 27 -\N 0 -\N -1 -\N -2 -\N 0 -\N -1 -\N -2 -0 0 -2 3 -4 6 -6 9 -8 12 -10 15 -12 18 -14 21 -16 24 -18 27 -12 18 -14 21 -16 24 -18 27 -0 0 -2 3 -4 6 -6 9 -8 12 diff --git a/tests/queries/0_stateless/01410_nullable_key.sql b/tests/queries/0_stateless/01410_nullable_key.sql deleted file mode 100644 index 4a3701cf46d..00000000000 --- a/tests/queries/0_stateless/01410_nullable_key.sql +++ /dev/null @@ -1,13 +0,0 @@ -DROP TABLE IF EXISTS nullable_key; -CREATE TABLE nullable_key (k Nullable(int), v int) ENGINE MergeTree ORDER BY k SETTINGS allow_nullable_key = 1; - -INSERT INTO nullable_key SELECT number * 2, number * 3 FROM numbers(10); -INSERT INTO nullable_key SELECT NULL, -number FROM numbers(3); - -SELECT * FROM nullable_key ORDER BY k; -SELECT * FROM nullable_key WHERE k IS NULL; -SELECT * FROM nullable_key WHERE k IS NOT NULL; -SELECT * FROM nullable_key WHERE k > 10; -SELECT * FROM nullable_key WHERE k < 10; - -DROP TABLE nullable_key; diff --git a/tests/queries/0_stateless/01410_nullable_key_and_index.reference b/tests/queries/0_stateless/01410_nullable_key_and_index.reference new file mode 100644 index 00000000000..1fc2cf91e62 --- /dev/null +++ b/tests/queries/0_stateless/01410_nullable_key_and_index.reference @@ -0,0 +1,81 @@ +0 0 +2 3 +4 6 +6 9 +8 12 +10 15 +12 18 +14 21 +16 24 +18 27 +\N 0 +\N -1 +\N -2 +\N 0 +\N -1 +\N -2 +0 0 +2 3 +4 6 +6 9 +8 12 +10 15 +12 18 +14 21 +16 24 +18 27 +12 18 +14 21 +16 24 +18 27 +0 0 +2 3 +4 6 +6 9 +8 12 +\N 0 +\N -1 +\N -2 +0 0 +2 3 +4 6 +6 9 +8 12 +10 15 +12 18 +14 21 +16 24 +18 27 +10 15 +\N 0 +\N -1 +\N -2 +\N +123 +1 1 +1 3 +2 \N +2 2 +2 1 +2 7 +2 \N +3 \N +3 2 +3 4 +2 \N +2 \N +3 \N +1 3 +2 7 +3 4 +1 1 +2 2 +2 1 +3 2 +1 3 +2 7 +3 4 +1 1 +2 2 +2 1 +3 2 diff --git a/tests/queries/0_stateless/01410_nullable_key_and_index.sql b/tests/queries/0_stateless/01410_nullable_key_and_index.sql new file mode 100644 index 00000000000..ba473b5c29a --- /dev/null +++ b/tests/queries/0_stateless/01410_nullable_key_and_index.sql @@ -0,0 +1,61 @@ +DROP TABLE IF EXISTS nullable_key; +DROP TABLE IF EXISTS nullable_key_without_final_mark; +DROP TABLE IF EXISTS nullable_minmax_index; + +SET max_threads = 1; + +CREATE TABLE nullable_key (k Nullable(int), v int) ENGINE MergeTree ORDER BY k SETTINGS allow_nullable_key = 1, index_granularity = 1; + +INSERT INTO nullable_key SELECT number * 2, number * 3 FROM numbers(10); +INSERT INTO nullable_key SELECT NULL, -number FROM numbers(3); + +SELECT * FROM nullable_key ORDER BY k; + +SET force_primary_key = 1; +SET max_rows_to_read = 3; +SELECT * FROM nullable_key WHERE k IS NULL; +SET max_rows_to_read = 10; +SELECT * FROM nullable_key WHERE k IS NOT NULL; +SET max_rows_to_read = 5; +SELECT * FROM nullable_key WHERE k > 10; +SELECT * FROM nullable_key WHERE k < 10; + +OPTIMIZE TABLE nullable_key FINAL; + +SET max_rows_to_read = 4; -- one additional left mark needs to be read +SELECT * FROM nullable_key WHERE k IS NULL; +SET max_rows_to_read = 10; +SELECT * FROM nullable_key WHERE k IS NOT NULL; + +-- Nullable in set and with transform_null_in = 1 +SET max_rows_to_read = 3; +SELECT * FROM nullable_key WHERE k IN (10, 20) SETTINGS transform_null_in = 1; +SET max_rows_to_read = 5; +SELECT * FROM nullable_key WHERE k IN (3, NULL) SETTINGS transform_null_in = 1; + +CREATE TABLE nullable_key_without_final_mark (s Nullable(String)) ENGINE MergeTree ORDER BY s SETTINGS allow_nullable_key = 1, write_final_mark = 0; +INSERT INTO nullable_key_without_final_mark VALUES ('123'), (NULL); +SET max_rows_to_read = 0; +SELECT * FROM nullable_key_without_final_mark WHERE s IS NULL; +SELECT * FROM nullable_key_without_final_mark WHERE s IS NOT NULL; + +CREATE TABLE nullable_minmax_index (k int, v Nullable(int), INDEX v_minmax v TYPE minmax GRANULARITY 4) ENGINE MergeTree ORDER BY k SETTINGS index_granularity = 1; + +INSERT INTO nullable_minmax_index VALUES (1, 3), (2, 7), (3, 4), (2, NULL); -- [3, +Inf] +INSERT INTO nullable_minmax_index VALUES (1, 1), (2, 2), (3, 2), (2, 1); -- [1, 2] +INSERT INTO nullable_minmax_index VALUES (2, NULL), (3, NULL); -- [+Inf, +Inf] + +SET force_primary_key = 0; +SELECT * FROM nullable_minmax_index ORDER BY k; +SET max_rows_to_read = 6; +SELECT * FROM nullable_minmax_index WHERE v IS NULL; +SET max_rows_to_read = 8; +SELECT * FROM nullable_minmax_index WHERE v IS NOT NULL; +SET max_rows_to_read = 6; +SELECT * FROM nullable_minmax_index WHERE v > 2; +SET max_rows_to_read = 4; +SELECT * FROM nullable_minmax_index WHERE v <= 2; + +DROP TABLE nullable_key; +DROP TABLE nullable_key_without_final_mark; +DROP TABLE nullable_minmax_index; diff --git a/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.reference b/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.reference new file mode 100644 index 00000000000..de0116f9eaa --- /dev/null +++ b/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.reference @@ -0,0 +1,20 @@ +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +0 2 +1 3 +\N 100 +\N 100 +\N 100 +\N 100 diff --git a/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.sql b/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.sql new file mode 100644 index 00000000000..72f12ce435a --- /dev/null +++ b/tests/queries/0_stateless/01585_use_index_for_global_in_with_null.sql @@ -0,0 +1,30 @@ +drop table if exists xp; +drop table if exists xp_d; + +create table xp(i Nullable(UInt64), j UInt64) engine MergeTree order by i settings index_granularity = 1, allow_nullable_key = 1; +create table xp_d as xp engine Distributed(test_shard_localhost, currentDatabase(), xp); + +insert into xp select number, number + 2 from numbers(10); +insert into xp select null, 100; + +optimize table xp final; + +set max_rows_to_read = 2; +select * from xp where i in (select * from numbers(2)); +select * from xp where i global in (select * from numbers(2)); +select * from xp_d where i in (select * from numbers(2)); +select * from xp_d where i global in (select * from numbers(2)); + +set transform_null_in = 1; +select * from xp where i in (select * from numbers(2)); +select * from xp where i global in (select * from numbers(2)); +select * from xp_d where i in (select * from numbers(2)); +select * from xp_d where i global in (select * from numbers(2)); + +select * from xp where i in (null); +select * from xp where i global in (null); +select * from xp_d where i in (null); +select * from xp_d where i global in (null); + +drop table if exists xp; +drop table if exists xp_d;