diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index 80093a06063..64380e5cd53 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -25,7 +25,7 @@ inline constexpr size_t MAX_ARRAY_SIZE = 1 << 30; /// Methods to copy Slice to Sink, overloaded for various combinations of types. template -void writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) +void ALWAYS_INLINE writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) { sink.elements.resize(sink.current_offset + slice.size); memcpySmallAllowReadWriteOverflow15(&sink.elements[sink.current_offset], slice.data, slice.size * sizeof(T)); @@ -33,7 +33,7 @@ void writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) } template -void writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) +void ALWAYS_INLINE writeSlice(const NumericArraySlice & slice, NumericArraySink & sink) { using NativeU = typename NativeType::Type; @@ -146,7 +146,7 @@ inline ALWAYS_INLINE void writeSlice(const Slice & slice, NullableArraySink -void writeSlice(const NumericValueSlice & slice, NumericArraySink & sink) +void ALWAYS_INLINE writeSlice(const NumericValueSlice & slice, NumericArraySink & sink) { sink.elements.resize(sink.current_offset + 1); sink.elements[sink.current_offset] = slice.value; @@ -203,7 +203,7 @@ void ALWAYS_INLINE concat(SourceA && src_a, SourceB && src_b, Sink && sink) } template -void concat(const std::vector> & array_sources, Sink && sink) +void ALWAYS_INLINE concat(const std::vector> & array_sources, Sink && sink) { size_t sources_num = array_sources.size(); std::vector is_const(sources_num); @@ -423,7 +423,7 @@ template < typename FirstSliceType, typename SecondSliceType, bool (*isEqual)(const FirstSliceType &, const SecondSliceType &, size_t, size_t)> -bool sliceHasImplAnyAll(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) +bool ALWAYS_INLINE sliceHasImplAnyAll(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) { const bool has_first_null_map = first_null_map != nullptr; const bool has_second_null_map = second_null_map != nullptr; @@ -457,7 +457,7 @@ bool sliceHasImplAnyAll(const FirstSliceType & first, const SecondSliceType & se /// https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm. /// A "prefix-function" is defined as: i-th element is the length of the longest of all prefixes that end in i-th position template -std::vector buildKMPPrefixFunction(const SliceType & pattern, const EqualityFunc & isEqualFunc) +std::vector ALWAYS_INLINE buildKMPPrefixFunction(const SliceType & pattern, const EqualityFunc & isEqualFunc) { std::vector result(pattern.size); result[0] = 0; @@ -484,7 +484,7 @@ template < typename FirstSliceType, typename SecondSliceType, bool (*isEqual)(const FirstSliceType &, const SecondSliceType &, size_t, size_t), bool (*isEqualUnary)(const SecondSliceType &, size_t, size_t)> -bool sliceHasImplSubstr(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) +bool ALWAYS_INLINE sliceHasImplSubstr(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) { if (second.size == 0) return true; @@ -541,7 +541,7 @@ template < typename SecondSliceType, bool (*isEqual)(const FirstSliceType &, const SecondSliceType &, size_t, size_t), bool (*isEqualSecond)(const SecondSliceType &, size_t, size_t)> -bool sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) +bool ALWAYS_INLINE sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) { if constexpr (search_type == ArraySearchType::Substr) return sliceHasImplSubstr(first, second, first_null_map, second_null_map); @@ -551,7 +551,7 @@ bool sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, template -bool sliceEqualElements(const NumericArraySlice & first [[maybe_unused]], +bool ALWAYS_INLINE sliceEqualElements(const NumericArraySlice & first [[maybe_unused]], const NumericArraySlice & second [[maybe_unused]], size_t first_ind [[maybe_unused]], size_t second_ind [[maybe_unused]]) @@ -566,13 +566,13 @@ bool sliceEqualElements(const NumericArraySlice & first [[maybe_unused]], } template -bool sliceEqualElements(const NumericArraySlice &, const GenericArraySlice &, size_t, size_t) +bool ALWAYS_INLINE sliceEqualElements(const NumericArraySlice &, const GenericArraySlice &, size_t, size_t) { return false; } template -bool sliceEqualElements(const GenericArraySlice &, const NumericArraySlice &, size_t, size_t) +bool ALWAYS_INLINE sliceEqualElements(const GenericArraySlice &, const NumericArraySlice &, size_t, size_t) { return false; } @@ -583,7 +583,7 @@ inline ALWAYS_INLINE bool sliceEqualElements(const GenericArraySlice & first, co } template -bool insliceEqualElements(const NumericArraySlice & first [[maybe_unused]], +bool ALWAYS_INLINE insliceEqualElements(const NumericArraySlice & first [[maybe_unused]], size_t first_ind [[maybe_unused]], size_t second_ind [[maybe_unused]]) { @@ -598,14 +598,14 @@ inline ALWAYS_INLINE bool insliceEqualElements(const GenericArraySlice & first, } template -bool sliceHas(const NumericArraySlice & first, const NumericArraySlice & second) +bool ALWAYS_INLINE sliceHas(const NumericArraySlice & first, const NumericArraySlice & second) { auto impl = sliceHasImpl, NumericArraySlice, sliceEqualElements, insliceEqualElements>; return impl(first, second, nullptr, nullptr); } template -bool sliceHas(const GenericArraySlice & first, const GenericArraySlice & second) +bool ALWAYS_INLINE sliceHas(const GenericArraySlice & first, const GenericArraySlice & second) { /// Generic arrays should have the same type in order to use column.compareAt(...) if (!first.elements->structureEquals(*second.elements)) @@ -616,19 +616,19 @@ bool sliceHas(const GenericArraySlice & first, const GenericArraySlice & second) } template -bool sliceHas(const GenericArraySlice & /*first*/, const NumericArraySlice & /*second*/) +bool ALWAYS_INLINE sliceHas(const GenericArraySlice & /*first*/, const NumericArraySlice & /*second*/) { return false; } template -bool sliceHas(const NumericArraySlice & /*first*/, const GenericArraySlice & /*second*/) +bool ALWAYS_INLINE sliceHas(const NumericArraySlice & /*first*/, const GenericArraySlice & /*second*/) { return false; } template -bool sliceHas(const FirstArraySlice & first, NullableSlice & second) +bool ALWAYS_INLINE sliceHas(const FirstArraySlice & first, NullableSlice & second) { auto impl = sliceHasImpl< search_type, @@ -640,7 +640,7 @@ bool sliceHas(const FirstArraySlice & first, NullableSlice & s } template -bool sliceHas(const NullableSlice & first, SecondArraySlice & second) +bool ALWAYS_INLINE sliceHas(const NullableSlice & first, SecondArraySlice & second) { auto impl = sliceHasImpl< search_type, @@ -652,7 +652,7 @@ bool sliceHas(const NullableSlice & first, SecondArraySlice & s } template -bool sliceHas(const NullableSlice & first, NullableSlice & second) +bool ALWAYS_INLINE sliceHas(const NullableSlice & first, NullableSlice & second) { auto impl = sliceHasImpl< search_type, @@ -677,7 +677,7 @@ void ALWAYS_INLINE arrayAllAny(FirstSource && first, SecondSource && second, Col } template -void resizeDynamicSize(ArraySource && array_source, ValueSource && value_source, Sink && sink, const IColumn & size_column) +void ALWAYS_INLINE resizeDynamicSize(ArraySource && array_source, ValueSource && value_source, Sink && sink, const IColumn & size_column) { const auto * size_nullable = typeid_cast(&size_column); const NullMap * size_null_map = size_nullable ? &size_nullable->getNullMapData() : nullptr; @@ -736,7 +736,7 @@ void resizeDynamicSize(ArraySource && array_source, ValueSource && value_source, } template -void resizeConstantSize(ArraySource && array_source, ValueSource && value_source, Sink && sink, const ssize_t size) +void ALWAYS_INLINE resizeConstantSize(ArraySource && array_source, ValueSource && value_source, Sink && sink, const ssize_t size) { while (!sink.isEnd()) {