Merge pull request #2756 from CurtizJ/CLICKHOUSE-3785

Clickhouse 3785 Add MurmurHash2

CMakeLists.txt

@@ -283,6 +283,7 @@ include (cmake/find_contrib_lib.cmake)
 find_contrib_lib(cityhash)
 find_contrib_lib(farmhash)
 find_contrib_lib(metrohash)
+find_contrib_lib(murmurhash2)
 find_contrib_lib(btrie)
 find_contrib_lib(double-conversion)
 

contrib/CMakeLists.txt

@@ -45,6 +45,10 @@ if (USE_INTERNAL_UNWIND_LIBRARY)
     add_subdirectory (libunwind)
 endif ()
 
+if (USE_INTERNAL_MURMURHASH2_LIBRARY)
+    add_subdirectory (libmurmurhash2)
+endif ()
+
 if (USE_INTERNAL_ZLIB_LIBRARY)
     add_subdirectory (${INTERNAL_ZLIB_NAME})
     # todo: make pull to Dead2/zlib-ng and remove:

contrib/libmurmurhash2/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_library(murmurhash2
+	src/murmurhash2.cpp
+	include/murmurhash2.h)
+
+target_include_directories (murmurhash2 PUBLIC include)
+target_include_directories (murmurhash2 PUBLIC src)

contrib/libmurmurhash2/LICENSE

@@ -0,0 +1 @@
+MurmurHash2 was written by Austin Appleby, and is placed in the publicdomain. The author hereby disclaims copyright to this source code.

contrib/libmurmurhash2/README

@@ -0,0 +1,6 @@
+Original URL: https://github.com/aappleby/smhasher
+
+version:
+commit  61a0530f28277f2e850bfc39600ce61d02b518de 
+author  aappleby@gmail.com
+date    2016-01-09T06:07:17Z

contrib/libmurmurhash2/include/murmurhash2.h

@@ -0,0 +1,35 @@
+//-----------------------------------------------------------------------------
+// MurmurHash2 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+#ifndef _MURMURHASH2_H_
+#define _MURMURHASH2_H_
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER) && (_MSC_VER < 1600)
+
+typedef unsigned char uint8_t;
+typedef unsigned int uint32_t;
+typedef unsigned __int64 uint64_t;
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#include <stdint.h>
+
+#endif // !defined(_MSC_VER)
+
+uint32_t MurmurHash2        (const void * key, int len, uint32_t seed);
+uint64_t MurmurHash64A      (const void * key, int len, uint64_t seed);
+uint64_t MurmurHash64B      (const void * key, int len, uint64_t seed);
+uint32_t MurmurHash2A       (const void * key, int len, uint32_t seed);
+uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed);
+uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed);
+
+#endif // _MURMURHASH2_H_
+

contrib/libmurmurhash2/src/murmurhash2.cpp

@@ -0,0 +1,421 @@
+// MurmurHash2 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+// Note - This code makes a few assumptions about how your machine behaves -
+
+// 1. We can read a 4-byte value from any address without crashing
+// 2. sizeof(int) == 4
+
+// And it has a few limitations -
+
+// 1. It will not work incrementally.
+// 2. It will not produce the same results on little-endian and big-endian
+//    machines.
+
+#include "murmurhash2.h"
+
+// Platform-specific functions and macros
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+
+#define BIG_CONSTANT(x) (x)
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+#endif // !defined(_MSC_VER)
+
+
+uint32_t MurmurHash2(const void * key, int len, uint32_t seed)
+{
+    // 'm' and 'r' are mixing constants generated offline.
+    // They're not really 'magic', they just happen to work well.
+
+    const uint32_t m = 0x5bd1e995;
+    const int r = 24;
+
+    // Initialize the hash to a 'random' value
+
+    uint32_t h = seed ^ len;
+
+    // Mix 4 bytes at a time into the hash
+
+    const unsigned char * data = reinterpret_cast<const unsigned char *>(key);
+
+    while (len >= 4)
+    {
+        uint32_t k = *reinterpret_cast<const uint32_t *>(data);
+        k *= m;
+        k ^= k >> r;
+        k *= m;
+
+        h *= m;
+        h ^= k;
+
+        data += 4;
+        len -= 4;
+    }
+
+    // Handle the last few bytes of the input array
+ 
+    switch (len)
+    {
+        case 3: h ^= data[2] << 16;
+        case 2: h ^= data[1] << 8;
+        case 1: h ^= data[0];
+                h *= m;
+    };
+
+    // Do a few final mixes of the hash to ensure the last few
+    // bytes are well-incorporated.
+
+    h ^= h >> 13;
+    h *= m;
+    h ^= h >> 15;
+
+    return h;
+} 
+
+// MurmurHash2, 64-bit versions, by Austin Appleby
+
+// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment 
+// and endian-ness issues if used across multiple platforms.
+
+// 64-bit hash for 64-bit platforms
+
+uint64_t MurmurHash64A(const void * key, int len, uint64_t seed)
+{
+    const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
+    const int r = 47;
+
+    uint64_t h = seed ^ (len * m);
+
+    const uint64_t * data = reinterpret_cast<const uint64_t *>(key);
+    const uint64_t * end = data + (len/8);
+
+    while (data != end)
+    {
+        uint64_t k = *data++;
+
+        k *= m; 
+        k ^= k >> r; 
+        k *= m; 
+
+        h ^= k;
+        h *= m; 
+    }
+
+    const unsigned char * data2 = reinterpret_cast<const unsigned char *>(data);
+
+    switch (len & 7)
+    {
+        case 7: h ^= static_cast<uint64_t>(data2[6]) << 48;
+        case 6: h ^= static_cast<uint64_t>(data2[5]) << 40;
+        case 5: h ^= static_cast<uint64_t>(data2[4]) << 32;
+        case 4: h ^= static_cast<uint64_t>(data2[3]) << 24;
+        case 3: h ^= static_cast<uint64_t>(data2[2]) << 16;
+        case 2: h ^= static_cast<uint64_t>(data2[1]) << 8;
+        case 1: h ^= static_cast<uint64_t>(data2[0]);
+                h *= m;
+    };
+
+    h ^= h >> r;
+    h *= m;
+    h ^= h >> r;
+
+    return h;
+} 
+
+
+// 64-bit hash for 32-bit platforms
+
+uint64_t MurmurHash64B(const void * key, int len, uint64_t seed)
+{
+    const uint32_t m = 0x5bd1e995;
+    const int r = 24;
+
+    uint32_t h1 = static_cast<uint32_t>(seed) ^ len;
+    uint32_t h2 = static_cast<uint32_t>(seed >> 32);
+
+    const uint32_t * data = reinterpret_cast<const uint32_t *>(key);
+
+    while (len >= 8)
+    {
+        uint32_t k1 = *data++;
+        k1 *= m; k1 ^= k1 >> r; k1 *= m;
+        h1 *= m; h1 ^= k1;
+        len -= 4;
+
+        uint32_t k2 = *data++;
+        k2 *= m; k2 ^= k2 >> r; k2 *= m;
+        h2 *= m; h2 ^= k2;
+        len -= 4;
+    }
+
+    if (len >= 4)
+    {
+        uint32_t k1 = *data++;
+        k1 *= m; k1 ^= k1 >> r; k1 *= m;
+        h1 *= m; h1 ^= k1;
+        len -= 4;
+    }
+
+    switch (len)
+    {
+        case 3: h2 ^= reinterpret_cast<const unsigned char *>(data)[2] << 16;
+        case 2: h2 ^= reinterpret_cast<const unsigned char *>(data)[1] << 8;
+        case 1: h2 ^= reinterpret_cast<const unsigned char *>(data)[0];
+                h2 *= m;
+    };
+
+    h1 ^= h2 >> 18; h1 *= m;
+    h2 ^= h1 >> 22; h2 *= m;
+    h1 ^= h2 >> 17; h1 *= m;
+    h2 ^= h1 >> 19; h2 *= m;
+
+    uint64_t h = h1;
+
+    h = (h << 32) | h2;
+
+    return h;
+} 
+
+// MurmurHash2A, by Austin Appleby
+
+// This is a variant of MurmurHash2 modified to use the Merkle-Damgard 
+// construction. Bulk speed should be identical to Murmur2, small-key speed 
+// will be 10%-20% slower due to the added overhead at the end of the hash.
+
+// This variant fixes a minor issue where null keys were more likely to
+// collide with each other than expected, and also makes the function
+// more amenable to incremental implementations.
+
+#define mmix(h,k) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; }
+
+uint32_t MurmurHash2A(const void * key, int len, uint32_t seed)
+{
+    const uint32_t m = 0x5bd1e995;
+    const int r = 24;
+    uint32_t l = len;
+
+    const unsigned char * data = reinterpret_cast<const unsigned char *>(key);
+
+    uint32_t h = seed;
+
+    while (len >= 4)
+    {
+        uint32_t k = *reinterpret_cast<const uint32_t *>(data);
+        mmix(h,k);
+        data += 4;
+        len -= 4;
+    }
+
+    uint32_t t = 0;
+
+    switch (len)
+    {
+        case 3: t ^= data[2] << 16;
+        case 2: t ^= data[1] << 8;
+        case 1: t ^= data[0];
+    };
+
+    mmix(h,t);
+    mmix(h,l);
+
+    h ^= h >> 13;
+    h *= m;
+    h ^= h >> 15;
+
+    return h;
+}
+
+// MurmurHashNeutral2, by Austin Appleby
+
+// Same as MurmurHash2, but endian- and alignment-neutral.
+// Half the speed though, alas.
+
+uint32_t MurmurHashNeutral2(const void * key, int len, uint32_t seed)
+{
+    const uint32_t m = 0x5bd1e995;
+    const int r = 24;
+
+    uint32_t h = seed ^ len;
+
+    const unsigned char * data = reinterpret_cast<const unsigned char *>(key);
+
+    while (len >= 4)
+    {
+        uint32_t k;
+
+        k  = data[0];
+        k |= data[1] << 8;
+        k |= data[2] << 16;
+        k |= data[3] << 24;
+
+        k *= m; 
+        k ^= k >> r; 
+        k *= m;
+
+        h *= m;
+        h ^= k;
+
+        data += 4;
+        len -= 4;
+    }
+
+    switch (len)
+    {
+        case 3: h ^= data[2] << 16;
+        case 2: h ^= data[1] << 8;
+        case 1: h ^= data[0];
+                h *= m;
+    };
+
+    h ^= h >> 13;
+    h *= m;
+    h ^= h >> 15;
+
+    return h;
+} 
+
+//-----------------------------------------------------------------------------
+// MurmurHashAligned2, by Austin Appleby
+
+// Same algorithm as MurmurHash2, but only does aligned reads - should be safer
+// on certain platforms. 
+
+// Performance will be lower than MurmurHash2
+
+#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; }
+
+
+uint32_t MurmurHashAligned2(const void * key, int len, uint32_t seed)
+{
+    const uint32_t m = 0x5bd1e995;
+    const int r = 24;
+
+    const unsigned char * data = reinterpret_cast<const unsigned char *>(key);
+
+    uint32_t h = seed ^ len;
+
+    int align = reinterpret_cast<uint64_t>(data) & 3;
+
+    if (align && (len >= 4))
+    {
+        // Pre-load the temp registers
+
+        uint32_t t = 0, d = 0;
+
+        switch (align)
+        {
+            case 1: t |= data[2] << 16;
+            case 2: t |= data[1] << 8;
+            case 3: t |= data[0];
+        }
+
+        t <<= (8 * align);
+
+        data += 4-align;
+        len -= 4-align;
+
+        int sl = 8 * (4-align);
+        int sr = 8 * align;
+
+        // Mix
+
+        while (len >= 4)
+        {
+            d = *(reinterpret_cast<const uint32_t *>(data));
+            t = (t >> sr) | (d << sl);
+
+            uint32_t k = t;
+
+            MIX(h,k,m);
+
+            t = d;
+
+            data += 4;
+            len -= 4;
+        }
+
+        // Handle leftover data in temp registers
+
+        d = 0;
+
+        if (len >= align)
+        {
+            switch (align)
+            {
+                case 3: d |= data[2] << 16;
+                case 2: d |= data[1] << 8;
+                case 1: d |= data[0];
+            }
+
+            uint32_t k = (t >> sr) | (d << sl);
+            MIX(h,k,m);
+
+            data += align;
+            len -= align;
+
+            //----------
+            // Handle tail bytes
+
+            switch (len)
+            {
+                case 3: h ^= data[2] << 16;
+                case 2: h ^= data[1] << 8;
+                case 1: h ^= data[0];
+                        h *= m;
+            };
+        }
+        else
+        {
+            switch (len)
+            {
+                case 3: d |= data[2] << 16;
+                case 2: d |= data[1] << 8;
+                case 1: d |= data[0];
+                case 0: h ^= (t >> sr) | (d << sl);
+                        h *= m;
+            }
+        }
+
+        h ^= h >> 13;
+        h *= m;
+        h ^= h >> 15;
+
+        return h;
+    }
+    else
+    {
+        while (len >= 4)
+        {
+            uint32_t k = *reinterpret_cast<const uint32_t *>(data);
+
+            MIX(h,k,m);
+
+            data += 4;
+            len -= 4;
+        }
+
+        // Handle tail bytes
+
+        switch (len)
+        {
+            case 3: h ^= data[2] << 16;
+            case 2: h ^= data[1] << 8;
+            case 1: h ^= data[0];
+                    h *= m;
+        };
+
+        h ^= h >> 13;
+        h *= m;
+        h ^= h >> 15;
+
+        return h;
+  }
+}

dbms/src/Functions/CMakeLists.txt

@@ -91,7 +91,7 @@ list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h Func
 
 add_library(clickhouse_functions ${clickhouse_functions_sources})
 
-target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE libconsistent-hashing ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES})
+target_link_libraries(clickhouse_functions PUBLIC dbms PRIVATE libconsistent-hashing ${FARMHASH_LIBRARIES} ${METROHASH_LIBRARIES} ${MURMURHASH2_LIBRARIES})
 
 target_include_directories (clickhouse_functions SYSTEM BEFORE PUBLIC ${DIVIDE_INCLUDE_DIR})
 

dbms/src/Functions/FunctionsHashing.cpp

@@ -20,6 +20,7 @@ void registerFunctionsHashing(FunctionFactory & factory)
     factory.registerFunction<FunctionIntHash32>();
     factory.registerFunction<FunctionIntHash64>();
     factory.registerFunction<FunctionURLHash>();
+    factory.registerFunction<FunctionMurmurHash2>();
 }
 
 template <>

dbms/src/Functions/FunctionsHashing.h

@@ -5,6 +5,7 @@
 #include <city.h>
 #include <farmhash.h>
 #include <metrohash.h>
+#include <murmurhash2.h>
 
 #include <Poco/ByteOrder.h>
 
@@ -143,6 +144,7 @@ struct SipHash64Impl
     }
 };
 
+
 struct SipHash128Impl
 {
     static constexpr auto name = "sipHash128";
@@ -366,7 +368,6 @@ UInt64 toInteger<Float32>(Float32 x);
 template <>
 UInt64 toInteger<Float64>(Float64 x);
 
-
 /** We use hash functions called CityHash, FarmHash, MetroHash.
   * In this regard, this template is named with the words `NeighborhoodHash`.
   */
@@ -614,6 +615,120 @@ public:
 };
 
 
+template <typename Impl>
+class FunctionStringHash32 : public IFunction
+{
+public:
+    static constexpr auto name = Impl::name;
+    static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHash32>(); }
+
+    String getName() const override { return name; }
+
+    bool isVariadic() const override { return false; }
+
+    size_t getNumberOfArguments() const override { return 1; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override { return std::make_shared<DataTypeUInt32>(); }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
+    {
+        auto col_to = ColumnUInt32::create(input_rows_count);
+        ColumnUInt32::Container & vec_to = col_to->getData();
+
+        const ColumnWithTypeAndName & col = block.getByPosition(arguments[0]);
+        const IDataType * from_type = col.type.get();
+        const IColumn * icolumn = col.column.get();
+
+        if      (checkDataType<DataTypeUInt8>(from_type)) executeIntType<UInt8>(icolumn, vec_to);
+        else if (checkDataType<DataTypeUInt16>(from_type)) executeIntType<UInt16>(icolumn, vec_to);
+        else if (checkDataType<DataTypeUInt32>(from_type)) executeIntType<UInt32>(icolumn, vec_to);
+        else if (checkDataType<DataTypeUInt64>(from_type)) executeIntType<UInt64>(icolumn, vec_to);
+        else if (checkDataType<DataTypeInt8>(from_type)) executeIntType<Int8>(icolumn, vec_to);
+        else if (checkDataType<DataTypeInt16>(from_type)) executeIntType<Int16>(icolumn, vec_to);
+        else if (checkDataType<DataTypeInt32>(from_type)) executeIntType<Int32>(icolumn, vec_to);
+        else if (checkDataType<DataTypeInt64>(from_type)) executeIntType<Int64>(icolumn, vec_to);
+        else if (checkDataType<DataTypeEnum8>(from_type)) executeIntType<Int8>(icolumn, vec_to);
+        else if (checkDataType<DataTypeEnum16>(from_type)) executeIntType<Int16>(icolumn, vec_to);
+        else if (checkDataType<DataTypeDate>(from_type)) executeIntType<UInt16>(icolumn, vec_to);
+        else if (checkDataType<DataTypeDateTime>(from_type)) executeIntType<UInt32>(icolumn, vec_to);
+        else if (checkDataType<DataTypeFloat32>(from_type)) executeIntType<Float32>(icolumn, vec_to);
+        else if (checkDataType<DataTypeFloat64>(from_type)) executeIntType<Float64>(icolumn, vec_to);
+        else if (checkDataType<DataTypeString>(from_type)) executeString(icolumn, vec_to);
+        else if (checkDataType<DataTypeFixedString>(from_type)) executeString(icolumn, vec_to);
+        else
+            throw Exception("Unexpected type " + from_type->getName() + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+        block.getByPosition(result).column = std::move(col_to);
+    }
+private:
+    template <typename FromType>
+    void executeIntType(const IColumn * column, ColumnUInt32::Container & vec_to)
+    {
+        if (const ColumnVector<FromType> * col_from = checkAndGetColumn<ColumnVector<FromType>>(column))
+        {
+            const typename ColumnVector<FromType>::Container & vec_from = col_from->getData();
+            size_t size = vec_from.size();
+            for (size_t i = 0; i < size; ++i)
+            {
+                vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&vec_from[i]), sizeof(FromType));
+            }
+        }
+        else
+            throw Exception("Illegal column " + column->getName()
+                + " of argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+    }
+
+    void executeString(const IColumn * column, ColumnUInt32::Container & vec_to)
+    {
+        if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(column))
+        {
+            const typename ColumnString::Chars_t & data = col_from->getChars();
+            const typename ColumnString::Offsets & offsets = col_from->getOffsets();
+            size_t size = offsets.size();
+            for (size_t i = 0; i < size; ++i)
+            {
+                vec_to[i] = Impl::Hash32(
+                    reinterpret_cast<const char *>(&data[i == 0 ? 0 : offsets[i - 1]]),
+                    i == 0 ? offsets[i] - 1 : (offsets[i] - 1 - offsets[i - 1]));
+            }
+        }
+        else if (const ColumnFixedString * col_from = checkAndGetColumn<ColumnFixedString>(column))
+        {
+            const typename ColumnString::Chars_t & data = col_from->getChars();
+            size_t n = col_from->getN();
+            size_t size = data.size() / n;
+            for (size_t i = 0; i < size; ++i)
+                vec_to[i] = Impl::Hash32(reinterpret_cast<const char *>(&data[i * n]), n);
+        }
+        else
+            throw Exception("Illegal column " + column->getName()
+                    + " of first argument of function " + getName(),
+                ErrorCodes::ILLEGAL_COLUMN);
+    }  
+};
+
+
+/** Why we need MurmurHash2?
+*   MurmurHash2 is an outdated hash function, superseded by MurmurHash3 and subsequently by CityHash, xxHash, HighwayHash. 
+*   Usually there is no reason to use MurmurHash.
+*   It is needed for the cases when you already have MurmurHash in some applications and you want to reproduce it 
+*   in ClickHouse as is. For example, it is needed to reproduce the behaviour 
+*   for NGINX a/b testing module: https://nginx.ru/en/docs/http/ngx_http_split_clients_module.html
+*/
+struct MurmurHash2Impl
+{
+    static constexpr auto name = "murmurHash2_32";
+    static UInt32 Hash32(const char * data, const size_t size) 
+    {
+        return MurmurHash2(data, size, 0);  
+    }
+};
+
+
 struct URLHashImpl
 {
     static UInt64 apply(const char * data, const size_t size)
@@ -848,5 +963,5 @@ using FunctionSipHash128 = FunctionStringHashFixedString<SipHash128Impl>;
 using FunctionCityHash64 = FunctionNeighbourhoodHash64<ImplCityHash64>;
 using FunctionFarmHash64 = FunctionNeighbourhoodHash64<ImplFarmHash64>;
 using FunctionMetroHash64 = FunctionNeighbourhoodHash64<ImplMetroHash64>;
-
+using FunctionMurmurHash2 = FunctionStringHash32<MurmurHash2Impl>;
 }

dbms/tests/queries/0_stateless/00678_murmurhash2.reference

@@ -0,0 +1,7 @@
+3012058918
+1298551497
+864444010
+3759658061
+2414502773
+670491991
+1343103100

dbms/tests/queries/0_stateless/00678_murmurhash2.sql

@@ -0,0 +1,9 @@
+SELECT murmurHash2(123456);
+SELECT murmurHash2(CAST(3 AS UInt8));
+SELECT murmurHash2(CAST(1.2684 AS Float32));
+SELECT murmurHash2(CAST(-154477 AS Int64));
+SELECT murmurHash2('foo');
+SELECT murmurHash2(CAST('bar' AS FixedString(3)));
+SELECT murmurHash2(x) FROM (SELECT CAST(1 AS Enum8('a' = 1, 'b' = 2)) as x);
+
+