ClickHouse/src/Functions/FunctionsHashing.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1772 lines
62 KiB
C++
Raw Normal View History

2012-07-15 21:43:04 +00:00
#pragma once
#include <city.h>
2015-07-17 15:56:08 +00:00
#include <farmhash.h>
2015-07-20 14:59:20 +00:00
#include <metrohash.h>
#include <wyhash.h>
2021-06-24 09:04:40 +00:00
#include <MurmurHash2.h>
#include <MurmurHash3.h>
#include "config.h"
2018-12-23 19:25:40 +00:00
#ifdef __clang__
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wused-but-marked-unused"
#endif
#include <xxhash.h>
#if USE_BLAKE3
# include <blake3.h>
#endif
2022-01-07 09:41:23 +00:00
2019-02-10 17:40:52 +00:00
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
#include <Common/safe_cast.h>
2019-02-10 17:40:52 +00:00
#include <Common/HashTable/Hash.h>
#if USE_SSL
# include <openssl/md4.h>
# include <openssl/md5.h>
# include <openssl/sha.h>
2018-12-23 19:25:40 +00:00
#endif
2012-07-15 21:43:04 +00:00
2023-03-16 17:03:43 +00:00
#include <bit>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeFixedString.h>
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeTuple.h>
2022-07-28 19:12:00 +00:00
#include <DataTypes/DataTypeMap.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnTuple.h>
2022-07-28 19:12:00 +00:00
#include <Columns/ColumnMap.h>
2021-05-17 07:30:42 +00:00
#include <Functions/IFunction.h>
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
#include <Functions/FunctionHelpers.h>
#include <Functions/PerformanceAdaptors.h>
2022-04-04 12:07:05 +00:00
#include <Common/TargetSpecific.h>
2023-01-08 20:58:09 +00:00
#include <base/IPv4andIPv6.h>
2021-10-02 07:13:14 +00:00
#include <base/range.h>
#include <base/bit_cast.h>
2023-02-16 19:03:41 +00:00
#include <base/unaligned.h>
2012-07-15 21:43:04 +00:00
namespace DB
{
2017-06-13 02:06:53 +00:00
namespace ErrorCodes
{
2020-02-25 18:02:41 +00:00
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
2017-06-13 02:06:53 +00:00
extern const int LOGICAL_ERROR;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
2018-12-18 20:24:16 +00:00
extern const int NOT_IMPLEMENTED;
extern const int ILLEGAL_COLUMN;
extern const int SUPPORT_IS_DISABLED;
2017-06-13 02:06:53 +00:00
}
namespace impl
{
struct SipHashKey
{
UInt64 key0 = 0;
UInt64 key1 = 0;
};
static SipHashKey parseSipHashKey(const ColumnWithTypeAndName & key)
{
SipHashKey ret;
const auto * tuple = checkAndGetColumn<ColumnTuple>(key.column.get());
if (!tuple)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "key must be a tuple");
if (tuple->tupleSize() != 2)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "wrong tuple size: key must be a tuple of 2 UInt64");
if (const auto * key0col = checkAndGetColumn<ColumnUInt64>(&(tuple->getColumn(0))))
ret.key0 = key0col->get64(0);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "first element of the key tuple is not UInt64");
if (const auto * key1col = checkAndGetColumn<ColumnUInt64>(&(tuple->getColumn(1))))
ret.key1 = key1col->get64(0);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "second element of the key tuple is not UInt64");
return ret;
}
}
2017-06-13 02:06:53 +00:00
2017-05-27 15:45:25 +00:00
/** Hashing functions.
2012-07-15 21:43:04 +00:00
*
* halfMD5: String -> UInt64
2012-07-15 21:43:04 +00:00
*
2017-05-27 15:45:25 +00:00
* A faster cryptographic hash function:
* sipHash64: String -> UInt64
*
2017-05-27 15:45:25 +00:00
* Fast non-cryptographic hash function for strings:
2012-07-15 21:43:04 +00:00
* cityHash64: String -> UInt64
*
2018-10-23 17:43:09 +00:00
* A non-cryptographic hashes from a tuple of values of any types (uses respective function for strings and intHash64 for numbers):
* cityHash64: any* -> UInt64
2018-10-23 17:43:09 +00:00
* sipHash64: any* -> UInt64
* halfMD5: any* -> UInt64
*
2017-05-27 15:45:25 +00:00
* Fast non-cryptographic hash function from any integer:
* intHash32: number -> UInt32
* intHash64: number -> UInt64
*
2012-07-15 21:43:04 +00:00
*/
2022-09-09 10:27:29 +00:00
2018-10-23 17:43:09 +00:00
struct IntHash32Impl
{
using ReturnType = UInt32;
static UInt32 apply(UInt64 x)
{
/// seed is taken from /dev/urandom. It allows you to avoid undesirable dependencies with hashes in different data structures.
return intHash32<0x75D9543DE018BF45ULL>(x);
}
};
struct IntHash64Impl
{
using ReturnType = UInt64;
static UInt64 apply(UInt64 x)
{
return intHash64(x ^ 0x4CF2D2BAAE6DA887ULL);
}
};
template<typename T, typename HashFunction>
T combineHashesFunc(T t1, T t2)
{
2023-03-08 15:38:52 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
T tmp;
reverseMemcpy(&tmp, &t1, sizeof(T));
t1 = tmp;
reverseMemcpy(&tmp, &t2, sizeof(T));
t2 = tmp;
#endif
T hashes[] = {t1, t2};
return HashFunction::apply(reinterpret_cast<const char *>(hashes), 2 * sizeof(T));
}
#if USE_SSL
2012-07-15 21:43:04 +00:00
struct HalfMD5Impl
{
2018-10-23 17:43:09 +00:00
static constexpr auto name = "halfMD5";
using ReturnType = UInt64;
2012-07-15 21:43:04 +00:00
static UInt64 apply(const char * begin, size_t size)
{
union
{
unsigned char char_data[16];
Squashed commit of the following: commit 2722e540abfee4a527d716474c4ca582eceeee08 Merge: b4f3af824 aebddd550 Author: proller <proller@github.com> Date: Tue Aug 21 18:34:18 2018 +0300 Merge remote-tracking branch 'upstream/master' into fix3 commit b4f3af824b88a8b6826583bb483730827574e8ad Author: proller <proller@github.com> Date: Tue Aug 21 17:00:20 2018 +0300 fix commit 3a18fa7ded2a7d6b7a0883a1b9c2d6b99360837d Author: proller <proller@github.com> Date: Tue Aug 21 16:57:17 2018 +0300 fix commit 5d42220c2cf47e1a86bdc73dbfc24b68f8626891 Author: proller <proller@github.com> Date: Tue Aug 21 16:50:21 2018 +0300 freebsd fixes commit 7baf4fb5ba4575f79b0d12a9ffaaabd34e1006da Author: proller <proller@github.com> Date: Tue Aug 21 16:17:19 2018 +0300 fix commit e1fe707fd765b841a8d0952d91a980128cbf91d0 Author: proller <proller@github.com> Date: Tue Aug 21 15:35:21 2018 +0300 fix commit 027887c71b3ffa98d9473d50d8c3b79cbf3304ac Author: proller <proller@github.com> Date: Tue Aug 21 15:25:57 2018 +0300 fix commit 81af41bfcfff7c02fe4060196cf03c2d2aab416e Author: proller <proller@github.com> Date: Tue Aug 21 15:20:41 2018 +0300 fix commit 93d572c85d9c7e331254999a614c3b22b5573b02 Author: proller <proller@github.com> Date: Tue Aug 21 14:58:07 2018 +0300 SPLIT_SHARED commit dd5b8990d8527d59b1e890943d80c2bf27c613ce Author: proller <proller@github.com> Date: Tue Aug 21 14:52:39 2018 +0300 fix commit 4840ca12ab752a38c1ef482e8ec59c5859bb48d7 Author: proller <proller@github.com> Date: Tue Aug 21 14:46:31 2018 +0300 fix commit abde633beb86f2a0d025d6fcf079965dbd827b92 Author: proller <proller@github.com> Date: Tue Aug 21 14:25:32 2018 +0300 fix commit 0d94a5476a5ba8ba5e88638d58f2cfbf2b4b662d Author: proller <proller@github.com> Date: Tue Aug 21 14:45:18 2018 +0300 Travis: try fail on ninja fail commit c2686f90b68255c2beb0a708804aef404e80a6d2 Merge: 2c3427bbd 2aa7eb463 Author: proller <proller@github.com> Date: Tue Aug 21 14:25:15 2018 +0300 Merge remote-tracking branch 'upstream/master' into fix3 commit 2c3427bbdb861edbb188ed4621e8a05cafaedafb Author: proller <proller@github.com> Date: Mon Aug 20 23:57:24 2018 +0300 fix commit 530170c6a81d31dcfa81230e48520383234df4bc Merge: 9abec162c f6e4ec970 Author: proller <proller@github.com> Date: Mon Aug 20 23:57:03 2018 +0300 Merge remote-tracking branch 'upstream/master' into fix3 commit 9abec162cb2e09bbc2f33cbe80fe76791f6e5a77 Author: proller <proller@github.com> Date: Mon Aug 20 23:49:58 2018 +0300 Apple fixes commit 36d05e8217440fbc8ae21571b06d4eb6d679d538 Author: proller <proller@github.com> Date: Mon Aug 20 23:25:05 2018 +0300 apple fix commit aeec3e845e4456e89fbb1b1af6f9f36820a46e33 Author: proller <proller@github.com> Date: Mon Aug 20 23:20:06 2018 +0300 fixes commit 427961d916a5954981e47d94733996deb2a616ce Author: proller <proller@github.com> Date: Mon Aug 20 23:11:11 2018 +0300 fix commit a7dd55ff8f653624c0f3dbcbc54defd3b3ae97af Author: proller <proller@github.com> Date: Mon Aug 20 22:41:53 2018 +0300 fix commit 6200e0d315c7a62bae63a8de0fc32f7937770ad2 Merge: 8a541d7e6 21cedbe46 Author: proller <proller@github.com> Date: Mon Aug 20 22:35:49 2018 +0300 Merge remote-tracking branch 'upstream/master' into fix3 commit 8a541d7e64c89e2c16af6c909e0353361153aaa3 Author: proller <proller@github.com> Date: Mon Aug 20 22:34:32 2018 +0300 Do not use poco types commit fd560f43d048b7e3307c6c6b9c9d9918230014d8 Author: proller <proller@github.com> Date: Mon Aug 20 22:20:42 2018 +0300 Try fix apple build commit cfb2eba07ac06f19e822d3474341d800b1f98cf1 Merge: 8d2e31c90 5b81fdfc0 Author: proller <proller@github.com> Date: Mon Aug 20 22:03:24 2018 +0300 Merge remote-tracking branch 'upstream/master' into fix3 commit 8d2e31c908be2e99d09e2a9dde2414ab82a5e93c Author: proller <proller@github.com> Date: Mon Aug 20 21:56:06 2018 +0300 travis: TEST_SERVER_STARTUP_WAIT=10
2018-08-21 15:56:50 +00:00
uint64_t uint64_data;
2012-07-15 21:43:04 +00:00
} buf;
2012-07-15 21:43:04 +00:00
MD5_CTX ctx;
MD5_Init(&ctx);
MD5_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
MD5_Final(buf.char_data, &ctx);
2023-03-01 21:36:35 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
return buf.uint64_data; /// No need to flip bytes on big endian machines
#else
2023-03-16 17:03:43 +00:00
return std::byteswap(buf.uint64_data); /// Compatibility with existing code. Cast need for old poco AND macos where UInt64 != uint64_t
2023-03-01 21:36:35 +00:00
#endif
2012-07-15 21:43:04 +00:00
}
2018-10-23 17:43:09 +00:00
2018-10-24 13:12:59 +00:00
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
2018-10-23 17:43:09 +00:00
{
2023-03-08 15:38:52 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
2023-03-16 17:03:43 +00:00
h1 = std::byteswap(h1);
h2 = std::byteswap(h2);
2023-03-08 15:38:52 +00:00
#endif
2018-10-24 13:12:59 +00:00
UInt64 hashes[] = {h1, h2};
return apply(reinterpret_cast<const char *>(hashes), 16);
2018-10-23 17:43:09 +00:00
}
2018-11-01 15:47:08 +00:00
/// If true, it will use intHash32 or intHash64 to hash POD types. This behaviour is intended for better performance of some functions.
/// Otherwise it will hash bytes in memory as a string using corresponding hash function.
2018-12-21 17:53:16 +00:00
2018-11-01 15:47:08 +00:00
static constexpr bool use_int_hash_for_pods = false;
2012-07-15 21:43:04 +00:00
};
struct MD4Impl
{
static constexpr auto name = "MD4";
enum { length = MD4_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
MD4_CTX ctx;
MD4_Init(&ctx);
MD4_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
MD4_Final(out_char_data, &ctx);
}
};
struct MD5Impl
{
static constexpr auto name = "MD5";
enum { length = MD5_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
MD5_CTX ctx;
MD5_Init(&ctx);
MD5_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
MD5_Final(out_char_data, &ctx);
}
};
struct SHA1Impl
{
static constexpr auto name = "SHA1";
enum { length = SHA_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
SHA_CTX ctx;
SHA1_Init(&ctx);
SHA1_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
SHA1_Final(out_char_data, &ctx);
}
};
struct SHA224Impl
{
static constexpr auto name = "SHA224";
enum { length = SHA224_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
SHA256_CTX ctx;
SHA224_Init(&ctx);
SHA224_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
SHA224_Final(out_char_data, &ctx);
}
};
struct SHA256Impl
{
static constexpr auto name = "SHA256";
enum { length = SHA256_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
SHA256_Final(out_char_data, &ctx);
}
};
struct SHA384Impl
{
static constexpr auto name = "SHA384";
enum { length = SHA384_DIGEST_LENGTH };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
SHA512_CTX ctx;
SHA384_Init(&ctx);
SHA384_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
SHA384_Final(out_char_data, &ctx);
}
};
struct SHA512Impl
{
static constexpr auto name = "SHA512";
2021-08-19 07:56:47 +00:00
enum { length = 64 };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
SHA512_CTX ctx;
SHA512_Init(&ctx);
SHA512_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
SHA512_Final(out_char_data, &ctx);
}
};
#endif
struct SipHash64Impl
{
2018-10-23 17:43:09 +00:00
static constexpr auto name = "sipHash64";
using ReturnType = UInt64;
static UInt64 apply(const char * begin, size_t size)
{
return sipHash64(begin, size);
}
2018-10-23 17:43:09 +00:00
2018-10-24 13:12:59 +00:00
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
2018-10-23 18:07:20 +00:00
{
return combineHashesFunc<UInt64, SipHash64Impl>(h1, h2);
2018-10-23 17:43:09 +00:00
}
2018-11-01 15:47:08 +00:00
static constexpr bool use_int_hash_for_pods = false;
};
2018-07-30 12:19:22 +00:00
struct SipHash64KeyedImpl
{
static constexpr auto name = "sipHash64Keyed";
using ReturnType = UInt64;
using Key = impl::SipHashKey;
static Key parseKey(const ColumnWithTypeAndName & key) { return impl::parseSipHashKey(key); }
static UInt64 applyKeyed(const Key & key, const char * begin, size_t size) { return sipHash64Keyed(key.key0, key.key1, begin, size); }
static UInt64 combineHashesKeyed(const Key & key, UInt64 h1, UInt64 h2)
{
2023-03-08 15:38:52 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
2023-03-16 17:03:43 +00:00
h1 = std::byteswap(h1);
h2 = std::byteswap(h2);
2023-03-08 15:38:52 +00:00
#endif
UInt64 hashes[] = {h1, h2};
return applyKeyed(key, reinterpret_cast<const char *>(hashes), 2 * sizeof(UInt64));
}
static constexpr bool use_int_hash_for_pods = false;
};
struct SipHash128Impl
{
2014-11-10 11:37:48 +00:00
static constexpr auto name = "sipHash128";
using ReturnType = UInt128;
static UInt128 combineHashes(UInt128 h1, UInt128 h2)
{
return combineHashesFunc<UInt128, SipHash128Impl>(h1, h2);
}
static UInt128 apply(const char * data, const size_t size)
{
return sipHash128(data, size);
}
static constexpr bool use_int_hash_for_pods = false;
};
struct SipHash128KeyedImpl
{
static constexpr auto name = "sipHash128Keyed";
using ReturnType = UInt128;
using Key = impl::SipHashKey;
static Key parseKey(const ColumnWithTypeAndName & key) { return impl::parseSipHashKey(key); }
static UInt128 applyKeyed(const Key & key, const char * begin, size_t size) { return sipHash128Keyed(key.key0, key.key1, begin, size); }
static UInt128 combineHashesKeyed(const Key & key, UInt128 h1, UInt128 h2)
{
2023-03-08 15:38:52 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
UInt128 tmp;
reverseMemcpy(&tmp, &h1, sizeof(UInt128));
h1 = tmp;
reverseMemcpy(&tmp, &h2, sizeof(UInt128));
h2 = tmp;
#endif
UInt128 hashes[] = {h1, h2};
return applyKeyed(key, reinterpret_cast<const char *>(hashes), 2 * sizeof(UInt128));
}
static constexpr bool use_int_hash_for_pods = false;
};
struct SipHash128ReferenceImpl
{
static constexpr auto name = "sipHash128Reference";
using ReturnType = UInt128;
static UInt128 combineHashes(UInt128 h1, UInt128 h2) { return combineHashesFunc<UInt128, SipHash128Impl>(h1, h2); }
static UInt128 apply(const char * data, const size_t size) { return sipHash128Reference(data, size); }
static constexpr bool use_int_hash_for_pods = false;
};
struct SipHash128ReferenceKeyedImpl
{
static constexpr auto name = "sipHash128ReferenceKeyed";
using ReturnType = UInt128;
using Key = impl::SipHashKey;
static Key parseKey(const ColumnWithTypeAndName & key) { return impl::parseSipHashKey(key); }
static UInt128 applyKeyed(const Key & key, const char * begin, size_t size)
{
return sipHash128ReferenceKeyed(key.key0, key.key1, begin, size);
}
static UInt128 combineHashesKeyed(const Key & key, UInt128 h1, UInt128 h2)
{
2023-03-08 15:38:52 +00:00
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
UInt128 tmp;
reverseMemcpy(&tmp, &h1, sizeof(UInt128));
h1 = tmp;
reverseMemcpy(&tmp, &h2, sizeof(UInt128));
h2 = tmp;
#endif
UInt128 hashes[] = {h1, h2};
return applyKeyed(key, reinterpret_cast<const char *>(hashes), 2 * sizeof(UInt128));
}
static constexpr bool use_int_hash_for_pods = false;
};
2018-11-01 15:47:08 +00:00
/** Why we need MurmurHash2?
* MurmurHash2 is an outdated hash function, superseded by MurmurHash3 and subsequently by CityHash, xxHash, HighwayHash.
* Usually there is no reason to use MurmurHash.
* It is needed for the cases when you already have MurmurHash in some applications and you want to reproduce it
* in ClickHouse as is. For example, it is needed to reproduce the behaviour
* for NGINX a/b testing module: https://nginx.ru/en/docs/http/ngx_http_split_clients_module.html
*/
struct MurmurHash2Impl32
{
static constexpr auto name = "murmurHash2_32";
using ReturnType = UInt32;
static UInt32 apply(const char * data, const size_t size)
{
return MurmurHash2(data, size, 0);
}
static UInt32 combineHashes(UInt32 h1, UInt32 h2)
{
return IntHash32Impl::apply(h1) ^ h2;
}
static constexpr bool use_int_hash_for_pods = false;
};
struct MurmurHash2Impl64
{
static constexpr auto name = "murmurHash2_64";
using ReturnType = UInt64;
static UInt64 apply(const char * data, const size_t size)
{
return MurmurHash64A(data, size, 0);
}
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
{
return IntHash64Impl::apply(h1) ^ h2;
}
static constexpr bool use_int_hash_for_pods = false;
};
2019-01-08 05:12:52 +00:00
/// To be compatible with gcc: https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191
2019-01-09 02:03:50 +00:00
struct GccMurmurHashImpl
2019-01-08 05:12:52 +00:00
{
2019-01-09 02:03:50 +00:00
static constexpr auto name = "gccMurmurHash";
2019-01-08 05:12:52 +00:00
using ReturnType = UInt64;
static UInt64 apply(const char * data, const size_t size)
{
return MurmurHash64A(data, size, 0xc70f6907UL);
}
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
{
return IntHash64Impl::apply(h1) ^ h2;
}
static constexpr bool use_int_hash_for_pods = false;
};
2023-03-30 23:07:49 +00:00
/// To be compatible with Default Partitioner in Kafka:
/// murmur2: https://github.com/apache/kafka/blob/461c5cfe056db0951d9b74f5adc45973670404d7/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L480
/// Default Partitioner: https://github.com/apache/kafka/blob/139f7709bd3f5926901a21e55043388728ccca78/clients/src/main/java/org/apache/kafka/clients/producer/internals/BuiltInPartitioner.java#L328
2023-03-29 18:05:25 +00:00
struct KafkaMurmurHashImpl
{
static constexpr auto name = "kafkaMurmurHash";
2023-03-30 23:07:49 +00:00
using ReturnType = UInt32;
2023-03-29 18:05:25 +00:00
2023-03-30 23:07:49 +00:00
static UInt32 apply(const char * data, const size_t size)
2023-03-29 18:05:25 +00:00
{
2023-03-30 23:07:49 +00:00
return MurmurHash2(data, size, 0x9747b28cU) & 0x7fffffff;
2023-03-29 18:05:25 +00:00
}
2023-03-30 23:07:49 +00:00
static UInt32 combineHashes(UInt32 h1, UInt32 h2)
2023-03-29 18:05:25 +00:00
{
2023-03-30 23:07:49 +00:00
return IntHash32Impl::apply(h1) ^ h2;
2023-03-29 18:05:25 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
2018-11-01 15:47:08 +00:00
struct MurmurHash3Impl32
{
static constexpr auto name = "murmurHash3_32";
using ReturnType = UInt32;
static UInt32 apply(const char * data, const size_t size)
{
union
{
UInt32 h;
char bytes[sizeof(h)];
};
MurmurHash3_x86_32(data, size, 0, bytes);
return h;
}
static UInt32 combineHashes(UInt32 h1, UInt32 h2)
{
2021-10-11 03:50:43 +00:00
return IntHash32Impl::apply(h1) ^ h2;
2018-11-01 15:47:08 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
struct MurmurHash3Impl64
{
static constexpr auto name = "murmurHash3_64";
using ReturnType = UInt64;
static UInt64 apply(const char * data, const size_t size)
{
union
{
UInt64 h[2];
char bytes[16];
};
MurmurHash3_x64_128(data, size, 0, bytes);
return h[0] ^ h[1];
}
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
{
2021-10-11 03:50:43 +00:00
return IntHash64Impl::apply(h1) ^ h2;
2018-11-01 15:47:08 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
struct MurmurHash3Impl128
{
static constexpr auto name = "murmurHash3_128";
using ReturnType = UInt128;
static UInt128 apply(const char * data, const size_t size)
{
char bytes[16];
MurmurHash3_x64_128(data, size, 0, bytes);
return *reinterpret_cast<UInt128 *>(bytes);
}
2021-10-11 03:50:43 +00:00
static UInt128 combineHashes(UInt128 h1, UInt128 h2)
{
return combineHashesFunc<UInt128, MurmurHash3Impl128>(h1, h2);
}
static constexpr bool use_int_hash_for_pods = false;
};
2018-12-18 20:24:16 +00:00
/// Care should be taken to do all calculation in unsigned integers (to avoid undefined behaviour on overflow)
2020-06-27 19:05:00 +00:00
/// but obtain the same result as it is done in signed integers with two's complement arithmetic.
2018-12-18 20:24:16 +00:00
struct JavaHashImpl
{
static constexpr auto name = "javaHash";
using ReturnType = Int32;
2022-09-13 09:59:11 +00:00
static ReturnType apply(int64_t x)
{
2022-09-28 18:11:32 +00:00
return static_cast<ReturnType>(
2022-09-29 11:56:50 +00:00
static_cast<uint32_t>(x) ^ static_cast<uint32_t>(static_cast<uint64_t>(x) >> 32));
2022-09-13 09:59:11 +00:00
}
template <class T, typename std::enable_if<std::is_same_v<T, int8_t>
|| std::is_same_v<T, int16_t>
|| std::is_same_v<T, int32_t>, T>::type * = nullptr>
static ReturnType apply(T x)
{
return x;
}
template <typename T, typename std::enable_if<!std::is_same_v<T, int8_t>
&& !std::is_same_v<T, int16_t>
&& !std::is_same_v<T, int32_t>
&& !std::is_same_v<T, int64_t>, T>::type * = nullptr>
static ReturnType apply(T x)
{
2022-09-29 07:43:55 +00:00
if (std::is_unsigned_v<T>)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsigned types are not supported");
2022-09-13 09:59:11 +00:00
const size_t size = sizeof(T);
const char * data = reinterpret_cast<const char *>(&x);
return apply(data, size);
}
static ReturnType apply(const char * data, const size_t size)
2018-12-18 20:24:16 +00:00
{
UInt32 h = 0;
for (size_t i = 0; i < size; ++i)
h = 31 * h + static_cast<UInt32>(static_cast<Int8>(data[i]));
return static_cast<Int32>(h);
}
2022-09-13 09:59:11 +00:00
static ReturnType combineHashes(Int32, Int32)
2018-12-18 20:24:16 +00:00
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Java hash is not combineable for multiple arguments");
2018-12-18 20:24:16 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
2019-11-06 10:46:37 +00:00
struct JavaHashUTF16LEImpl
{
static constexpr auto name = "javaHashUTF16LE";
using ReturnType = Int32;
static Int32 apply(const char * raw_data, const size_t raw_size)
{
char * data = const_cast<char *>(raw_data);
size_t size = raw_size;
// Remove Byte-order-mark(0xFFFE) for UTF-16LE
if (size >= 2 && data[0] == '\xFF' && data[1] == '\xFE')
2019-11-06 10:46:37 +00:00
{
data += 2;
size -= 2;
}
if (size % 2 != 0)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Arguments for javaHashUTF16LE must be in the form of UTF-16");
2019-11-06 10:46:37 +00:00
UInt32 h = 0;
2019-11-06 10:46:37 +00:00
for (size_t i = 0; i < size; i += 2)
h = 31 * h + static_cast<UInt16>(static_cast<UInt8>(data[i]) | static_cast<UInt8>(data[i + 1]) << 8);
2019-11-06 10:46:37 +00:00
return static_cast<Int32>(h);
2019-11-06 10:46:37 +00:00
}
static Int32 combineHashes(Int32, Int32)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Java hash is not combineable for multiple arguments");
2019-11-06 10:46:37 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
2018-12-18 20:24:16 +00:00
/// This is just JavaHash with zeroed out sign bit.
/// This function is used in Hive for versions before 3.0,
/// after 3.0, Hive uses murmur-hash3.
struct HiveHashImpl
{
static constexpr auto name = "hiveHash";
using ReturnType = Int32;
static Int32 apply(const char * data, const size_t size)
{
return static_cast<Int32>(0x7FFFFFFF & static_cast<UInt32>(JavaHashImpl::apply(data, size)));
}
static Int32 combineHashes(Int32, Int32)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Hive hash is not combineable for multiple arguments");
2018-12-18 20:24:16 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
2018-11-01 15:47:08 +00:00
struct ImplCityHash64
{
static constexpr auto name = "cityHash64";
using ReturnType = UInt64;
using uint128_t = CityHash_v1_0_2::uint128;
static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
static auto apply(const char * s, const size_t len) { return CityHash_v1_0_2::CityHash64(s, len); }
static constexpr bool use_int_hash_for_pods = true;
};
2020-10-31 12:45:53 +00:00
// see farmhash.h for definition of NAMESPACE_FOR_HASH_FUNCTIONS
struct ImplFarmFingerprint64
{
static constexpr auto name = "farmFingerprint64";
using ReturnType = UInt64;
using uint128_t = NAMESPACE_FOR_HASH_FUNCTIONS::uint128_t;
static auto combineHashes(UInt64 h1, UInt64 h2) { return NAMESPACE_FOR_HASH_FUNCTIONS::Fingerprint(uint128_t(h1, h2)); }
static auto apply(const char * s, const size_t len) { return NAMESPACE_FOR_HASH_FUNCTIONS::Fingerprint64(s, len); }
static constexpr bool use_int_hash_for_pods = true;
};
2018-11-01 15:47:08 +00:00
// see farmhash.h for definition of NAMESPACE_FOR_HASH_FUNCTIONS
struct ImplFarmHash64
{
static constexpr auto name = "farmHash64";
using ReturnType = UInt64;
using uint128_t = NAMESPACE_FOR_HASH_FUNCTIONS::uint128_t;
static auto combineHashes(UInt64 h1, UInt64 h2) { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash128to64(uint128_t(h1, h2)); }
static auto apply(const char * s, const size_t len) { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash64(s, len); }
static constexpr bool use_int_hash_for_pods = true;
};
struct ImplMetroHash64
{
static constexpr auto name = "metroHash64";
using ReturnType = UInt64;
using uint128_t = CityHash_v1_0_2::uint128;
static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
static auto apply(const char * s, const size_t len)
{
union
{
UInt64 u64;
2020-02-22 05:46:35 +00:00
uint8_t u8[sizeof(u64)];
2018-11-01 15:47:08 +00:00
};
2020-02-22 05:46:35 +00:00
metrohash64_1(reinterpret_cast<const uint8_t *>(s), len, 0, u8);
2018-11-01 15:47:08 +00:00
return u64;
}
static constexpr bool use_int_hash_for_pods = true;
};
2018-12-21 17:53:16 +00:00
struct ImplXxHash32
{
static constexpr auto name = "xxHash32";
using ReturnType = UInt32;
static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH32(s, len, 0); }
2018-12-21 17:53:16 +00:00
/**
* With current implementation with more than 1 arguments it will give the results
2020-08-08 00:47:03 +00:00
* non-reproducible from outside of CH.
2018-12-21 17:53:16 +00:00
*
* Proper way of combining several input is to use streaming mode of hash function
* https://github.com/Cyan4973/xxHash/issues/114#issuecomment-334908566
*
* In common case doable by init_state / update_state / finalize_state
*/
static auto combineHashes(UInt32 h1, UInt32 h2) { return IntHash32Impl::apply(h1) ^ h2; }
static constexpr bool use_int_hash_for_pods = false;
};
struct ImplXxHash64
{
static constexpr auto name = "xxHash64";
using ReturnType = UInt64;
using uint128_t = CityHash_v1_0_2::uint128;
static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH64(s, len, 0); }
/*
With current implementation with more than 1 arguments it will give the results
non-reproducible from outside of CH. (see comment on ImplXxHash32).
*/
static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
static constexpr bool use_int_hash_for_pods = false;
};
struct ImplXXH3
{
static constexpr auto name = "xxh3";
using ReturnType = UInt64;
using uint128_t = CityHash_v1_0_2::uint128;
static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH3_64bits(s, len); }
2018-12-21 17:53:16 +00:00
/*
With current implementation with more than 1 arguments it will give the results
2020-08-08 00:47:03 +00:00
non-reproducible from outside of CH. (see comment on ImplXxHash32).
2018-12-21 17:53:16 +00:00
*/
static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
static constexpr bool use_int_hash_for_pods = false;
};
2022-01-07 09:41:23 +00:00
struct ImplBLAKE3
{
static constexpr auto name = "BLAKE3";
2022-08-05 18:27:26 +00:00
enum { length = 32 };
2022-01-07 09:41:23 +00:00
#if !USE_BLAKE3
2022-10-06 18:41:26 +00:00
[[noreturn]] static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
{
UNUSED(begin);
UNUSED(size);
UNUSED(out_char_data);
2022-10-05 20:38:44 +00:00
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "BLAKE3 is not available. Rust code or BLAKE3 itself may be disabled.");
}
#else
2022-01-08 15:56:20 +00:00
static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
{
2022-02-13 10:12:45 +00:00
#if defined(MEMORY_SANITIZER)
auto err_msg = blake3_apply_shim_msan_compat(begin, safe_cast<uint32_t>(size), out_char_data);
2022-09-08 19:25:46 +00:00
__msan_unpoison(out_char_data, length);
2022-02-13 09:46:20 +00:00
#else
auto err_msg = blake3_apply_shim(begin, safe_cast<uint32_t>(size), out_char_data);
2022-02-13 09:46:20 +00:00
#endif
2022-01-27 16:51:11 +00:00
if (err_msg != nullptr)
{
auto err_st = std::string(err_msg);
blake3_free_char_pointer(err_msg);
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function returned error message: {}", err_st);
2022-01-12 15:28:30 +00:00
}
2022-01-07 09:41:23 +00:00
}
2022-10-05 20:38:44 +00:00
#endif
2022-01-07 09:41:23 +00:00
};
template <typename Impl>
class FunctionStringHashFixedString : public IFunction
{
public:
static constexpr auto name = Impl::name;
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionStringHashFixedString>(); }
String getName() const override
{
return name;
}
2016-12-29 19:38:10 +00:00
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!isStringOrFixedString(arguments[0]) && !isIPv6(arguments[0]))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
arguments[0]->getName(), getName());
return std::make_shared<DataTypeFixedString>(Impl::length);
}
bool useDefaultImplementationForConstants() const override { return true; }
2021-06-22 16:21:23 +00:00
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
2020-10-18 19:00:13 +00:00
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(arguments[0].column.get()))
{
auto col_to = ColumnFixedString::create(Impl::length);
const typename ColumnString::Chars & data = col_from->getChars();
const typename ColumnString::Offsets & offsets = col_from->getOffsets();
auto & chars_to = col_to->getChars();
const auto size = offsets.size();
chars_to.resize(size * Impl::length);
2018-07-30 18:00:16 +00:00
ColumnString::Offset current_offset = 0;
for (size_t i = 0; i < size; ++i)
2018-07-30 18:00:16 +00:00
{
Impl::apply(
2018-07-30 18:00:16 +00:00
reinterpret_cast<const char *>(&data[current_offset]),
offsets[i] - current_offset - 1,
2020-02-22 05:46:35 +00:00
reinterpret_cast<uint8_t *>(&chars_to[i * Impl::length]));
2018-07-30 18:00:16 +00:00
current_offset = offsets[i];
}
2020-10-18 19:00:13 +00:00
return col_to;
}
2020-06-23 05:35:35 +00:00
else if (
2020-10-18 19:00:13 +00:00
const ColumnFixedString * col_from_fix = checkAndGetColumn<ColumnFixedString>(arguments[0].column.get()))
2020-06-23 05:35:35 +00:00
{
auto col_to = ColumnFixedString::create(Impl::length);
const typename ColumnFixedString::Chars & data = col_from_fix->getChars();
const auto size = col_from_fix->size();
auto & chars_to = col_to->getChars();
const auto length = col_from_fix->getN();
chars_to.resize(size * Impl::length);
for (size_t i = 0; i < size; ++i)
{
Impl::apply(
reinterpret_cast<const char *>(&data[i * length]), length, reinterpret_cast<uint8_t *>(&chars_to[i * Impl::length]));
}
return col_to;
}
else if (
const ColumnIPv6 * col_from_ip = checkAndGetColumn<ColumnIPv6>(arguments[0].column.get()))
{
auto col_to = ColumnFixedString::create(Impl::length);
const typename ColumnIPv6::Container & data = col_from_ip->getData();
const auto size = col_from_ip->size();
auto & chars_to = col_to->getChars();
const auto length = IPV6_BINARY_LENGTH;
chars_to.resize(size * Impl::length);
for (size_t i = 0; i < size; ++i)
{
Impl::apply(
2020-06-23 05:35:35 +00:00
reinterpret_cast<const char *>(&data[i * length]), length, reinterpret_cast<uint8_t *>(&chars_to[i * Impl::length]));
}
2020-10-18 19:00:13 +00:00
return col_to;
2020-06-23 05:35:35 +00:00
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
arguments[0].column->getName(), getName());
}
};
DECLARE_MULTITARGET_CODE(
2012-07-15 21:43:04 +00:00
template <typename Impl, typename Name>
class FunctionIntHash : public IFunction
{
public:
static constexpr auto name = Name::name;
2012-07-15 21:43:04 +00:00
private:
using ToType = typename Impl::ReturnType;
2012-07-15 21:43:04 +00:00
template <typename FromType>
ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
2012-07-15 21:43:04 +00:00
{
2021-09-10 11:49:22 +00:00
using ColVecType = ColumnVectorOrDecimal<FromType>;
2020-10-18 19:00:13 +00:00
if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(arguments[0].column.get()))
2012-07-15 21:43:04 +00:00
{
auto col_to = ColumnVector<ToType>::create();
const typename ColVecType::Container & vec_from = col_from->getData();
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
2012-07-15 21:43:04 +00:00
size_t size = vec_from.size();
vec_to.resize(size);
for (size_t i = 0; i < size; ++i)
vec_to[i] = Impl::apply(vec_from[i]);
2020-10-18 19:00:13 +00:00
return col_to;
2012-07-15 21:43:04 +00:00
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
arguments[0].column->getName(), Name::name);
2012-07-15 21:43:04 +00:00
}
public:
String getName() const override
2012-07-15 21:43:04 +00:00
{
return name;
2012-07-15 21:43:04 +00:00
}
2016-12-29 19:38:10 +00:00
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
2012-07-15 21:43:04 +00:00
{
2018-11-01 15:47:08 +00:00
if (!arguments[0]->isValueRepresentedByNumber())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
arguments[0]->getName(), getName());
return std::make_shared<DataTypeNumber<typename Impl::ReturnType>>();
2012-07-15 21:43:04 +00:00
}
bool useDefaultImplementationForConstants() const override { return true; }
2021-06-22 16:21:23 +00:00
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
2012-07-15 21:43:04 +00:00
{
2020-10-18 19:00:13 +00:00
const IDataType * from_type = arguments[0].type.get();
WhichDataType which(from_type);
2020-10-18 19:00:13 +00:00
if (which.isUInt8())
return executeType<UInt8>(arguments);
else if (which.isUInt16())
return executeType<UInt16>(arguments);
else if (which.isUInt32())
return executeType<UInt32>(arguments);
else if (which.isUInt64())
return executeType<UInt64>(arguments);
else if (which.isInt8())
return executeType<Int8>(arguments);
else if (which.isInt16())
return executeType<Int16>(arguments);
else if (which.isInt32())
return executeType<Int32>(arguments);
else if (which.isInt64())
return executeType<Int64>(arguments);
else if (which.isDate())
return executeType<UInt16>(arguments);
else if (which.isDate32())
return executeType<Int32>(arguments);
2020-10-18 19:00:13 +00:00
else if (which.isDateTime())
return executeType<UInt32>(arguments);
else if (which.isDecimal32())
return executeType<Decimal32>(arguments);
else if (which.isDecimal64())
return executeType<Decimal64>(arguments);
2023-01-08 20:58:09 +00:00
else if (which.isIPv4())
return executeType<IPv4>(arguments);
2012-07-15 21:43:04 +00:00
else
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
arguments[0].type->getName(), getName());
2012-07-15 21:43:04 +00:00
}
};
) // DECLARE_MULTITARGET_CODE
template <typename Impl, typename Name>
class FunctionIntHash : public TargetSpecific::Default::FunctionIntHash<Impl, Name>
{
public:
explicit FunctionIntHash(ContextPtr context) : selector(context)
{
selector.registerImplementation<TargetArch::Default,
TargetSpecific::Default::FunctionIntHash<Impl, Name>>();
2020-05-28 11:48:56 +00:00
#if USE_MULTITARGET_CODE
selector.registerImplementation<TargetArch::AVX2,
TargetSpecific::AVX2::FunctionIntHash<Impl, Name>>();
selector.registerImplementation<TargetArch::AVX512F,
TargetSpecific::AVX512F::FunctionIntHash<Impl, Name>>();
#endif
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
2020-10-18 19:00:13 +00:00
return selector.selectAndExecute(arguments, result_type, input_rows_count);
}
2020-05-28 11:48:56 +00:00
static FunctionPtr create(ContextPtr context)
{
return std::make_shared<FunctionIntHash>(context);
}
private:
ImplementationSelector<IFunction> selector;
};
2012-07-15 21:43:04 +00:00
DECLARE_MULTITARGET_CODE(
template <typename Impl, bool Keyed, typename KeyType>
2018-10-23 17:43:09 +00:00
class FunctionAnyHash : public IFunction
{
public:
2015-07-17 15:56:08 +00:00
static constexpr auto name = Impl::name;
private:
2018-10-23 17:43:09 +00:00
using ToType = typename Impl::ReturnType;
2019-11-13 14:20:23 +00:00
template <typename FromType, bool first>
void executeIntType(const KeyType & key, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
{
2021-09-10 11:49:22 +00:00
using ColVecType = ColumnVectorOrDecimal<FromType>;
if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
{
const typename ColVecType::Container & vec_from = col_from->getData();
size_t size = vec_from.size();
for (size_t i = 0; i < size; ++i)
{
2018-10-23 17:43:09 +00:00
ToType h;
2018-11-01 15:47:08 +00:00
if constexpr (Impl::use_int_hash_for_pods)
{
2023-02-22 02:40:11 +00:00
if constexpr (std::is_same_v<ToType, UInt64>)
2023-03-01 21:36:35 +00:00
h = IntHash64Impl::apply(bit_cast<UInt64>(vec_from[i]));
2018-11-01 15:47:08 +00:00
else
2023-03-01 21:36:35 +00:00
h = IntHash32Impl::apply(bit_cast<UInt32>(vec_from[i]));
2018-11-01 15:47:08 +00:00
}
2018-10-23 17:43:09 +00:00
else
2018-11-01 15:47:08 +00:00
{
2023-02-22 20:50:46 +00:00
if constexpr (std::is_same_v<Impl, JavaHashImpl>)
2023-02-22 20:41:00 +00:00
h = JavaHashImpl::apply(vec_from[i]);
2022-09-13 09:59:11 +00:00
else
2023-02-16 19:03:41 +00:00
{
2023-02-22 23:19:25 +00:00
FromType v = vec_from[i];
if constexpr (std::endian::native == std::endian::big)
2023-02-22 20:41:00 +00:00
{
2023-02-22 23:19:25 +00:00
FromType tmp_v;
reverseMemcpy(&tmp_v, &v, sizeof(v));
v = tmp_v;
2023-02-16 19:03:41 +00:00
}
2023-02-22 23:19:25 +00:00
h = apply(key, reinterpret_cast<const char *>(&v), sizeof(v));
}
2018-11-01 15:47:08 +00:00
}
if constexpr (first)
vec_to[i] = h;
else
vec_to[i] = combineHashes(key, vec_to[i], h);
}
}
else if (auto col_from_const = checkAndGetColumnConst<ColVecType>(column))
{
auto value = col_from_const->template getValue<FromType>();
2018-10-23 17:43:09 +00:00
ToType hash;
2023-02-22 20:41:00 +00:00
if constexpr (std::is_same_v<ToType, UInt64>)
2023-03-01 21:36:35 +00:00
hash = IntHash64Impl::apply(bit_cast<UInt64>(value));
2018-10-23 17:43:09 +00:00
else
2023-03-01 21:36:35 +00:00
hash = IntHash32Impl::apply(bit_cast<UInt32>(value));
2023-02-22 20:41:00 +00:00
size_t size = vec_to.size();
if constexpr (first)
vec_to.assign(size, hash);
else
{
for (size_t i = 0; i < size; ++i)
vec_to[i] = combineHashes(key, vec_to[i], hash);
}
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
column->getName(), getName());
}
template <typename FromType, bool first>
void executeBigIntType(const KeyType & key, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
{
2021-09-10 11:49:22 +00:00
using ColVecType = ColumnVectorOrDecimal<FromType>;
if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
{
const typename ColVecType::Container & vec_from = col_from->getData();
size_t size = vec_from.size();
for (size_t i = 0; i < size; ++i)
{
2023-02-16 19:03:41 +00:00
ToType h;
if constexpr (std::endian::native == std::endian::little)
2023-02-25 07:46:21 +00:00
{
2023-02-16 19:03:41 +00:00
h = apply(key, reinterpret_cast<const char *>(&vec_from[i]), sizeof(vec_from[i]));
2023-02-25 07:46:21 +00:00
}
2023-02-16 19:03:41 +00:00
else
{
char tmp_buffer[sizeof(vec_from[i])];
reverseMemcpy(tmp_buffer, &vec_from[i], sizeof(vec_from[i]));
h = apply(key, reinterpret_cast<const char *>(tmp_buffer), sizeof(vec_from[i]));
}
if constexpr (first)
vec_to[i] = h;
else
vec_to[i] = combineHashes(key, vec_to[i], h);
}
}
else if (auto col_from_const = checkAndGetColumnConst<ColVecType>(column))
{
auto value = col_from_const->template getValue<FromType>();
2023-02-16 19:03:41 +00:00
ToType h;
if constexpr (std::endian::native == std::endian::little)
2023-02-25 07:46:21 +00:00
{
2023-02-16 19:03:41 +00:00
h = apply(key, reinterpret_cast<const char *>(&value), sizeof(value));
2023-02-25 07:46:21 +00:00
}
2023-02-16 19:03:41 +00:00
else
{
char tmp_buffer[sizeof(value)];
reverseMemcpy(tmp_buffer, &value, sizeof(value));
h = apply(key, reinterpret_cast<const char *>(tmp_buffer), sizeof(value));
}
size_t size = vec_to.size();
if constexpr (first)
vec_to.assign(size, h);
else
{
for (size_t i = 0; i < size; ++i)
vec_to[i] = combineHashes(key, vec_to[i], h);
}
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
column->getName(), getName());
}
2019-11-12 16:37:36 +00:00
template <bool first>
void executeGeneric(const KeyType & key, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
2019-12-06 15:21:58 +00:00
{
2019-12-06 19:21:22 +00:00
for (size_t i = 0, size = column->size(); i < size; ++i)
2019-12-06 15:21:58 +00:00
{
2019-12-06 16:27:45 +00:00
StringRef bytes = column->getDataAt(i);
const ToType h = apply(key, bytes.data, bytes.size);
if constexpr (first)
2019-12-06 16:27:45 +00:00
vec_to[i] = h;
2019-12-06 15:21:58 +00:00
else
vec_to[i] = combineHashes(key, vec_to[i], h);
2019-12-06 15:21:58 +00:00
}
}
template <bool first>
void executeString(const KeyType & key, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
{
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(column))
{
const typename ColumnString::Chars & data = col_from->getChars();
const typename ColumnString::Offsets & offsets = col_from->getOffsets();
size_t size = offsets.size();
2018-07-30 18:00:16 +00:00
ColumnString::Offset current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
const ToType h = apply(key,
2018-07-30 18:00:16 +00:00
reinterpret_cast<const char *>(&data[current_offset]),
offsets[i] - current_offset - 1);
if constexpr (first)
vec_to[i] = h;
else
vec_to[i] = combineHashes(key, vec_to[i], h);
2018-07-30 18:00:16 +00:00
current_offset = offsets[i];
}
}
else if (const ColumnFixedString * col_from_fixed = checkAndGetColumn<ColumnFixedString>(column))
{
const typename ColumnString::Chars & data = col_from_fixed->getChars();
size_t n = col_from_fixed->getN();
size_t size = data.size() / n;
2018-07-30 18:00:16 +00:00
for (size_t i = 0; i < size; ++i)
{
const ToType h = apply(key, reinterpret_cast<const char *>(&data[i * n]), n);
if constexpr (first)
vec_to[i] = h;
else
vec_to[i] = combineHashes(key, vec_to[i], h);
}
}
else if (const ColumnConst * col_from_const = checkAndGetColumnConstStringOrFixedString(column))
{
2020-10-18 19:00:13 +00:00
String value = col_from_const->getValue<String>();
const ToType hash = apply(key, value.data(), value.size());
2015-07-17 15:56:08 +00:00
const size_t size = vec_to.size();
2018-07-30 18:00:16 +00:00
if constexpr (first)
{
vec_to.assign(size, hash);
}
else
{
for (size_t i = 0; i < size; ++i)
{
vec_to[i] = combineHashes(key, vec_to[i], hash);
}
}
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
column->getName(), getName());
}
template <bool first>
void executeArray(const KeyType & key, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
{
2023-03-18 02:29:14 +00:00
const IDataType * nested_type = typeid_cast<const DataTypeArray &>(*type).getNestedType().get();
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
if (const ColumnArray * col_from = checkAndGetColumn<ColumnArray>(column))
{
const IColumn * nested_column = &col_from->getData();
const ColumnArray::Offsets & offsets = col_from->getOffsets();
2015-07-17 15:56:08 +00:00
const size_t nested_size = nested_column->size();
2018-10-23 17:43:09 +00:00
typename ColumnVector<ToType>::Container vec_temp(nested_size);
bool nested_is_first = true;
executeForArgument(key, nested_type, nested_column, vec_temp, nested_is_first);
2015-07-17 15:56:08 +00:00
const size_t size = offsets.size();
2018-07-30 18:00:16 +00:00
ColumnArray::Offset current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
2018-07-30 18:00:16 +00:00
ColumnArray::Offset next_offset = offsets[i];
2018-10-23 17:43:09 +00:00
ToType h;
if constexpr (std::is_same_v<ToType, UInt64>)
h = IntHash64Impl::apply(next_offset - current_offset);
else
h = IntHash32Impl::apply(next_offset - current_offset);
if constexpr (first)
vec_to[i] = h;
else
vec_to[i] = combineHashes(key, vec_to[i], h);
2018-07-30 18:00:16 +00:00
for (size_t j = current_offset; j < next_offset; ++j)
vec_to[i] = combineHashes(key, vec_to[i], vec_temp[j]);
2018-07-30 18:00:16 +00:00
current_offset = offsets[i];
}
}
else if (const ColumnConst * col_from_const = checkAndGetColumnConst<ColumnArray>(column))
{
2017-05-27 15:45:25 +00:00
/// NOTE: here, of course, you can do without the materialization of the column.
ColumnPtr full_column = col_from_const->convertToFullColumn();
2023-02-25 07:46:21 +00:00
executeArray<first>(key, type, full_column.get(), vec_to);
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
column->getName(), getName());
}
template <bool first>
void executeAny(const KeyType & key, const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to) const
{
WhichDataType which(from_type);
if (icolumn->size() != vec_to.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Argument column '{}' size {} doesn't match result column size {} of function {}",
icolumn->getName(), icolumn->size(), vec_to.size(), getName());
if (which.isUInt8()) executeIntType<UInt8, first>(key, icolumn, vec_to);
else if (which.isUInt16()) executeIntType<UInt16, first>(key, icolumn, vec_to);
else if (which.isUInt32()) executeIntType<UInt32, first>(key, icolumn, vec_to);
else if (which.isUInt64()) executeIntType<UInt64, first>(key, icolumn, vec_to);
else if (which.isUInt128()) executeBigIntType<UInt128, first>(key, icolumn, vec_to);
else if (which.isUInt256()) executeBigIntType<UInt256, first>(key, icolumn, vec_to);
else if (which.isInt8()) executeIntType<Int8, first>(key, icolumn, vec_to);
else if (which.isInt16()) executeIntType<Int16, first>(key, icolumn, vec_to);
else if (which.isInt32()) executeIntType<Int32, first>(key, icolumn, vec_to);
else if (which.isInt64()) executeIntType<Int64, first>(key, icolumn, vec_to);
else if (which.isInt128()) executeBigIntType<Int128, first>(key, icolumn, vec_to);
else if (which.isInt256()) executeBigIntType<Int256, first>(key, icolumn, vec_to);
else if (which.isUUID()) executeBigIntType<UUID, first>(key, icolumn, vec_to);
else if (which.isIPv4()) executeIntType<IPv4, first>(key, icolumn, vec_to);
else if (which.isIPv6()) executeBigIntType<IPv6, first>(key, icolumn, vec_to);
else if (which.isEnum8()) executeIntType<Int8, first>(key, icolumn, vec_to);
else if (which.isEnum16()) executeIntType<Int16, first>(key, icolumn, vec_to);
else if (which.isDate()) executeIntType<UInt16, first>(key, icolumn, vec_to);
else if (which.isDate32()) executeIntType<Int32, first>(key, icolumn, vec_to);
else if (which.isDateTime()) executeIntType<UInt32, first>(key, icolumn, vec_to);
/// TODO: executeIntType() for Decimal32/64 leads to incompatible result
else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(key, icolumn, vec_to);
else if (which.isDecimal64()) executeBigIntType<Decimal64, first>(key, icolumn, vec_to);
else if (which.isDecimal128()) executeBigIntType<Decimal128, first>(key, icolumn, vec_to);
else if (which.isDecimal256()) executeBigIntType<Decimal256, first>(key, icolumn, vec_to);
else if (which.isFloat32()) executeIntType<Float32, first>(key, icolumn, vec_to);
else if (which.isFloat64()) executeIntType<Float64, first>(key, icolumn, vec_to);
else if (which.isString()) executeString<first>(key, icolumn, vec_to);
else if (which.isFixedString()) executeString<first>(key, icolumn, vec_to);
else if (which.isArray()) executeArray<first>(key, from_type, icolumn, vec_to);
else executeGeneric<first>(key, icolumn, vec_to);
}
2019-11-13 15:18:24 +00:00
void executeForArgument(const KeyType & key, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first) const
{
/// Flattening of tuples.
if (const ColumnTuple * tuple = typeid_cast<const ColumnTuple *>(column))
{
const auto & tuple_columns = tuple->getColumns();
const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
size_t tuple_size = tuple_columns.size();
for (size_t i = 0; i < tuple_size; ++i)
executeForArgument(key, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
}
else if (const ColumnTuple * tuple_const = checkAndGetColumnConstData<ColumnTuple>(column))
2016-07-10 07:24:24 +00:00
{
const auto & tuple_columns = tuple_const->getColumns();
const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
size_t tuple_size = tuple_columns.size();
for (size_t i = 0; i < tuple_size; ++i)
{
auto tmp = ColumnConst::create(tuple_columns[i], column->size());
executeForArgument(key, tuple_types[i].get(), tmp.get(), vec_to, is_first);
}
2016-07-10 07:24:24 +00:00
}
2022-07-28 19:12:00 +00:00
else if (const auto * map = checkAndGetColumn<ColumnMap>(column))
{
const auto & type_map = assert_cast<const DataTypeMap &>(*type);
executeForArgument(key, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first);
2022-07-28 19:12:00 +00:00
}
2023-02-25 07:46:21 +00:00
else if (const auto * const_map = checkAndGetColumnConst<ColumnMap>(column))
2022-07-28 19:12:00 +00:00
{
2023-02-25 07:46:21 +00:00
executeForArgument(key, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first);
2022-07-28 19:12:00 +00:00
}
else
{
if (is_first)
executeAny<true>(key, type, column, vec_to);
else
executeAny<false>(key, type, column, vec_to);
}
is_first = false;
}
public:
String getName() const override
{
return name;
}
2016-12-29 19:38:10 +00:00
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
bool useDefaultImplementationForConstants() const override { return true; }
2021-06-22 16:21:23 +00:00
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
2017-12-02 02:47:12 +00:00
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
{
if constexpr (std::is_same_v<ToType, UInt128>) /// backward-compatible
{
return std::make_shared<DataTypeFixedString>(sizeof(UInt128));
}
else
return std::make_shared<DataTypeNumber<ToType>>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
2023-02-25 07:46:21 +00:00
auto col_to = ColumnVector<ToType>::create(input_rows_count);
2018-10-23 17:43:09 +00:00
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
/// If using a "keyed" algorithm, the first argument is the key and
/// the data starts from the second argument.
/// Otherwise there is no key and all arguments are interpreted as data.
constexpr size_t first_data_argument = Keyed;
if (arguments.size() <= first_data_argument)
{
/// Return a fixed random-looking magic number when input is empty
2023-02-25 07:46:21 +00:00
vec_to.assign(input_rows_count, static_cast<ToType>(0xe28dbde7fe22e41c));
}
KeyType key{};
if constexpr (Keyed)
if (!arguments.empty())
key = Impl::parseKey(arguments[0]);
/// The function supports arbitrary number of arguments of arbitrary types.
bool is_first_argument = true;
for (size_t i = first_data_argument; i < arguments.size(); ++i)
{
const auto & col = arguments[i];
executeForArgument(key, col.type.get(), col.column.get(), vec_to, is_first_argument);
}
if constexpr (std::is_same_v<ToType, UInt128>) /// backward-compatible
{
auto col_to_fixed_string = ColumnFixedString::create(sizeof(UInt128));
col_to_fixed_string->getChars() = std::move(*reinterpret_cast<ColumnFixedString::Chars *>(&col_to->getData()));
return col_to_fixed_string;
}
2020-10-18 19:00:13 +00:00
return col_to;
}
static ToType apply(const KeyType & key, const char * begin, size_t size)
{
if constexpr (Keyed)
return Impl::applyKeyed(key, begin, size);
else
return Impl::apply(begin, size);
}
static ToType combineHashes(const KeyType & key, ToType h1, ToType h2)
{
if constexpr (Keyed)
return Impl::combineHashesKeyed(key, h1, h2);
else
return Impl::combineHashes(h1, h2);
}
};
) // DECLARE_MULTITARGET_CODE
template <typename Impl, bool Keyed = false, typename KeyType = char>
class FunctionAnyHash : public TargetSpecific::Default::FunctionAnyHash<Impl, Keyed, KeyType>
{
public:
explicit FunctionAnyHash(ContextPtr context) : selector(context)
{
selector.registerImplementation<TargetArch::Default, TargetSpecific::Default::FunctionAnyHash<Impl, Keyed, KeyType>>();
#if USE_MULTITARGET_CODE
selector.registerImplementation<TargetArch::AVX2, TargetSpecific::AVX2::FunctionAnyHash<Impl, Keyed, KeyType>>();
selector.registerImplementation<TargetArch::AVX512F, TargetSpecific::AVX512F::FunctionAnyHash<Impl, Keyed, KeyType>>();
#endif
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
2020-10-18 19:00:13 +00:00
return selector.selectAndExecute(arguments, result_type, input_rows_count);
}
static FunctionPtr create(ContextPtr context)
{
return std::make_shared<FunctionAnyHash>(context);
}
private:
ImplementationSelector<IFunction> selector;
};
struct URLHashImpl
{
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
static UInt64 apply(const char * data, const size_t size)
{
/// do not take last slash, '?' or '#' character into account
if (size > 0 && (data[size - 1] == '/' || data[size - 1] == '?' || data[size - 1] == '#'))
2017-06-21 08:35:38 +00:00
return CityHash_v1_0_2::CityHash64(data, size - 1);
2017-06-21 08:35:38 +00:00
return CityHash_v1_0_2::CityHash64(data, size);
}
};
struct URLHierarchyHashImpl
{
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
static size_t findLevelLength(const UInt64 level, const char * begin, const char * end)
{
2020-10-18 19:00:13 +00:00
const auto * pos = begin;
2017-05-27 15:45:25 +00:00
/// Let's parse everything that goes before the path
2017-05-27 15:45:25 +00:00
/// Suppose that the protocol has already been changed to lowercase.
while (pos < end && ((*pos > 'a' && *pos < 'z') || (*pos > '0' && *pos < '9')))
++pos;
2017-05-27 15:45:25 +00:00
/** We will calculate the hierarchy only for URLs in which there is a protocol, and after it there are two slashes.
* (http, file - fit, mailto, magnet - do not fit), and after two slashes there is still something
* For the rest, simply return the full URL as the only element of the hierarchy.
*/
if (pos == begin || pos == end || !(*pos++ == ':' && pos < end && *pos++ == '/' && pos < end && *pos++ == '/' && pos < end))
{
pos = end;
return 0 == level ? pos - begin : 0;
}
2017-05-27 15:45:25 +00:00
/// The domain for simplicity is everything that after the protocol and the two slashes, until the next slash or before `?` or `#`
while (pos < end && !(*pos == '/' || *pos == '?' || *pos == '#'))
++pos;
if (pos != end)
++pos;
if (0 == level)
return pos - begin;
UInt64 current_level = 0;
while (current_level != level && pos < end)
{
2017-05-27 15:45:25 +00:00
/// We go to the next `/` or `?` or `#`, skipping all at the beginning.
while (pos < end && (*pos == '/' || *pos == '?' || *pos == '#'))
++pos;
if (pos == end)
break;
while (pos < end && !(*pos == '/' || *pos == '?' || *pos == '#'))
++pos;
if (pos != end)
++pos;
++current_level;
}
return current_level == level ? pos - begin : 0;
}
ColumnConst unification (#1011) * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * Fixed error in ColumnArray::replicateGeneric [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150]. * ColumnConst: unification (incomplete) [#CLICKHOUSE-3150].
2017-07-21 06:35:58 +00:00
static UInt64 apply(const UInt64 level, const char * data, const size_t size)
{
return URLHashImpl::apply(data, findLevelLength(level, data, data + size));
}
};
class FunctionURLHash : public IFunction
{
public:
static constexpr auto name = "URLHash";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionURLHash>(); }
String getName() const override { return name; }
2016-12-29 19:38:10 +00:00
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
2021-06-22 16:21:23 +00:00
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto arg_count = arguments.size();
if (arg_count != 1 && arg_count != 2)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: "
"passed {}, should be 1 or 2.", getName(), arg_count);
2020-10-18 19:00:13 +00:00
const auto * first_arg = arguments.front().get();
if (!WhichDataType(first_arg).isString())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", first_arg->getName(), getName());
if (arg_count == 2)
{
const auto & second_arg = arguments.back();
if (!isInteger(second_arg))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", second_arg->getName(), getName());
}
return std::make_shared<DataTypeUInt64>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
{
const auto arg_count = arguments.size();
if (arg_count == 1)
2020-10-18 19:00:13 +00:00
return executeSingleArg(arguments);
else if (arg_count == 2)
2020-10-18 19:00:13 +00:00
return executeTwoArgs(arguments);
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "got into IFunction::execute with unexpected number of arguments");
}
private:
ColumnPtr executeSingleArg(const ColumnsWithTypeAndName & arguments) const
{
2020-10-18 19:00:13 +00:00
const auto * col_untyped = arguments.front().column.get();
2020-10-18 19:00:13 +00:00
if (const auto * col_from = checkAndGetColumn<ColumnString>(col_untyped))
{
const auto size = col_from->size();
auto col_to = ColumnUInt64::create(size);
const auto & chars = col_from->getChars();
const auto & offsets = col_from->getOffsets();
auto & out = col_to->getData();
2018-07-30 18:00:16 +00:00
ColumnString::Offset current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
out[i] = URLHashImpl::apply(
2018-07-30 18:00:16 +00:00
reinterpret_cast<const char *>(&chars[current_offset]),
offsets[i] - current_offset - 1);
current_offset = offsets[i];
}
2020-10-18 19:00:13 +00:00
return col_to;
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
arguments[0].column->getName(), getName());
}
ColumnPtr executeTwoArgs(const ColumnsWithTypeAndName & arguments) const
{
2020-10-18 19:00:13 +00:00
const auto * level_col = arguments.back().column.get();
if (!isColumnConst(*level_col))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Second argument of function {} must be an integral constant", getName());
const auto level = level_col->get64(0);
2020-10-18 19:00:13 +00:00
const auto * col_untyped = arguments.front().column.get();
if (const auto * col_from = checkAndGetColumn<ColumnString>(col_untyped))
{
const auto size = col_from->size();
auto col_to = ColumnUInt64::create(size);
const auto & chars = col_from->getChars();
const auto & offsets = col_from->getOffsets();
auto & out = col_to->getData();
2018-07-30 18:00:16 +00:00
ColumnString::Offset current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
out[i] = URLHierarchyHashImpl::apply(
level,
reinterpret_cast<const char *>(&chars[current_offset]),
offsets[i] - current_offset - 1);
current_offset = offsets[i];
}
2020-10-18 19:00:13 +00:00
return col_to;
}
else
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
arguments[0].column->getName(), getName());
}
};
2022-04-20 23:26:37 +00:00
struct ImplWyHash64
{
static constexpr auto name = "wyHash64";
using ReturnType = UInt64;
static UInt64 apply(const char * s, const size_t len)
{
2022-04-20 23:31:31 +00:00
return wyhash(s, len, 0, _wyp);
2022-04-20 23:26:37 +00:00
}
static UInt64 combineHashes(UInt64 h1, UInt64 h2)
{
union
{
UInt64 u64[2];
2022-04-20 23:31:31 +00:00
char chars[16];
2022-04-20 23:26:37 +00:00
};
u64[0] = h1;
2022-04-20 23:31:31 +00:00
u64[1] = h2;
return apply(chars, 16);
2022-04-20 23:26:37 +00:00
}
static constexpr bool use_int_hash_for_pods = false;
};
2017-06-13 02:06:53 +00:00
struct NameIntHash32 { static constexpr auto name = "intHash32"; };
struct NameIntHash64 { static constexpr auto name = "intHash64"; };
2018-10-23 17:43:09 +00:00
using FunctionSipHash64 = FunctionAnyHash<SipHash64Impl>;
using FunctionSipHash64Keyed = FunctionAnyHash<SipHash64KeyedImpl, true, SipHash64KeyedImpl::Key>;
2015-07-17 15:56:08 +00:00
using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
#if USE_SSL
using FunctionMD4 = FunctionStringHashFixedString<MD4Impl>;
using FunctionHalfMD5 = FunctionAnyHash<HalfMD5Impl>;
2015-07-17 15:56:08 +00:00
using FunctionMD5 = FunctionStringHashFixedString<MD5Impl>;
using FunctionSHA1 = FunctionStringHashFixedString<SHA1Impl>;
using FunctionSHA224 = FunctionStringHashFixedString<SHA224Impl>;
using FunctionSHA256 = FunctionStringHashFixedString<SHA256Impl>;
using FunctionSHA384 = FunctionStringHashFixedString<SHA384Impl>;
using FunctionSHA512 = FunctionStringHashFixedString<SHA512Impl>;
#endif
using FunctionSipHash128 = FunctionAnyHash<SipHash128Impl>;
using FunctionSipHash128Keyed = FunctionAnyHash<SipHash128KeyedImpl, true, SipHash128KeyedImpl::Key>;
using FunctionSipHash128Reference = FunctionAnyHash<SipHash128ReferenceImpl>;
using FunctionSipHash128ReferenceKeyed = FunctionAnyHash<SipHash128ReferenceKeyedImpl, true, SipHash128ReferenceKeyedImpl::Key>;
2018-10-23 17:43:09 +00:00
using FunctionCityHash64 = FunctionAnyHash<ImplCityHash64>;
2020-10-31 12:45:53 +00:00
using FunctionFarmFingerprint64 = FunctionAnyHash<ImplFarmFingerprint64>;
2018-10-23 17:43:09 +00:00
using FunctionFarmHash64 = FunctionAnyHash<ImplFarmHash64>;
using FunctionMetroHash64 = FunctionAnyHash<ImplMetroHash64>;
2018-10-23 17:43:09 +00:00
using FunctionMurmurHash2_32 = FunctionAnyHash<MurmurHash2Impl32>;
using FunctionMurmurHash2_64 = FunctionAnyHash<MurmurHash2Impl64>;
2019-01-09 02:03:50 +00:00
using FunctionGccMurmurHash = FunctionAnyHash<GccMurmurHashImpl>;
2023-03-29 18:05:25 +00:00
using FunctionKafkaMurmurHash = FunctionAnyHash<KafkaMurmurHashImpl>;
2018-10-23 17:43:09 +00:00
using FunctionMurmurHash3_32 = FunctionAnyHash<MurmurHash3Impl32>;
using FunctionMurmurHash3_64 = FunctionAnyHash<MurmurHash3Impl64>;
using FunctionMurmurHash3_128 = FunctionAnyHash<MurmurHash3Impl128>;
2018-12-18 20:24:16 +00:00
using FunctionJavaHash = FunctionAnyHash<JavaHashImpl>;
2019-11-06 10:46:37 +00:00
using FunctionJavaHashUTF16LE = FunctionAnyHash<JavaHashUTF16LEImpl>;
2018-12-18 20:24:16 +00:00
using FunctionHiveHash = FunctionAnyHash<HiveHashImpl>;
2018-12-21 17:53:16 +00:00
2022-01-18 06:51:13 +00:00
using FunctionXxHash32 = FunctionAnyHash<ImplXxHash32>;
using FunctionXxHash64 = FunctionAnyHash<ImplXxHash64>;
using FunctionXXH3 = FunctionAnyHash<ImplXXH3>;
2018-12-23 19:25:40 +00:00
2022-04-20 23:26:37 +00:00
using FunctionWyHash64 = FunctionAnyHash<ImplWyHash64>;
2022-10-05 20:38:44 +00:00
using FunctionBLAKE3 = FunctionStringHashFixedString<ImplBLAKE3>;
2018-07-30 13:50:26 +00:00
}
#ifdef __clang__
# pragma clang diagnostic pop
#endif