thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 19:42:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge

Browse Source
This commit is contained in:
Alexey Arno 2015-07-20 19:17:34 +03:00
commit 1c9bcb6892
26 changed files with 13541 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
dbms/include/DB/DataStreams/TSKVRowOutputStream.h Normal file
View File

@ -0,0 +1,24 @@
#pragma once
#include <DB/DataStreams/TabSeparatedRowOutputStream.h>
namespace DB
{
/** Поток для вывода данных в формате TSKV.
* TSKV похож на TabSeparated, но перед каждым значением указывается его имя и знак равенства: name=value.
* Этот формат весьма неэффективен.
*/
class TSKVRowOutputStream : public TabSeparatedRowOutputStream
{
public:
TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_);
void writeField(const Field & field) override;
protected:
NamesAndTypes fields;
};
}

5
dbms/include/DB/Dictionaries/RangeHashedDictionary.h
View File

@ -17,8 +17,9 @@ namespace DB
class RangeHashedDictionary final : public IDictionaryBase
{
public:
RangeHashedDictionary(const std::string & name, const DictionaryStructure & dict_struct,
DictionarySourcePtr source_ptr, const DictionaryLifetime dict_lifetime)
RangeHashedDictionary(
const std::string & name, const DictionaryStructure & dict_struct, DictionarySourcePtr source_ptr,
const DictionaryLifetime dict_lifetime)
: name{name}, dict_struct(dict_struct),
source_ptr{std::move(source_ptr)}, dict_lifetime(dict_lifetime)
{

276
dbms/include/DB/Functions/FunctionsConditional.h
View File

@ -2,10 +2,13 @@
#include <DB/DataTypes/DataTypesNumberFixed.h>
#include <DB/DataTypes/DataTypeArray.h>
#include <DB/DataTypes/DataTypeString.h>
#include <DB/DataTypes/DataTypeFixedString.h>
#include <DB/Columns/ColumnVector.h>
#include <DB/Columns/ColumnString.h>
#include <DB/Columns/ColumnConst.h>
#include <DB/Columns/ColumnArray.h>
#include <DB/Columns/ColumnFixedString.h>
#include <DB/Functions/IFunction.h>
#include <DB/Functions/NumberTraits.h>
@ -170,7 +173,83 @@ struct StringIfImpl
}
}
static void vector_constant(
static void vector_fixed_vector_fixed(
const PODArray<UInt8> & cond,
const ColumnFixedString::Chars_t & a_data,
const ColumnFixedString::Chars_t & b_data,
const size_t N,
ColumnFixedString::Chars_t & c_data)
{
size_t size = cond.size();
c_data.resize(a_data.size());
for (size_t i = 0; i < size; ++i)
{
if (cond[i])
memcpy(&c_data[i * N], &a_data[i * N], N);
else
memcpy(&c_data[i * N], &b_data[i * N], N);
}
}
template <bool negative>
static void vector_vector_fixed_impl(
const PODArray<UInt8> & cond,
const ColumnString::Chars_t & a_data, const ColumnString::Offsets_t & a_offsets,
const ColumnFixedString::Chars_t & b_data, const size_t b_N,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
size_t size = cond.size();
c_offsets.resize(size);
c_data.reserve(std::max(a_data.size(), b_data.size() + size));
ColumnString::Offset_t a_prev_offset = 0;
ColumnString::Offset_t c_prev_offset = 0;
for (size_t i = 0; i < size; ++i)
{
if (negative != cond[i])
{
size_t size_to_write = a_offsets[i] - a_prev_offset;
c_data.resize(c_data.size() + size_to_write);
memcpy(&c_data[c_prev_offset], &a_data[a_prev_offset], size_to_write);
c_prev_offset += size_to_write;
c_offsets[i] = c_prev_offset;
}
else
{
size_t size_to_write = b_N;
c_data.resize(c_data.size() + size_to_write + 1);
memcpy(&c_data[c_prev_offset], &b_data[i * b_N], size_to_write);
c_data.back() = 0;
c_prev_offset += size_to_write + 1;
c_offsets[i] = c_prev_offset;
}
a_prev_offset = a_offsets[i];
}
}
static void vector_vector_fixed(
const PODArray<UInt8> & cond,
const ColumnString::Chars_t & a_data, const ColumnString::Offsets_t & a_offsets,
const ColumnFixedString::Chars_t & b_data, const size_t b_N,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
vector_vector_fixed_impl<false>(cond, a_data, a_offsets, b_data, b_N, c_data, c_offsets);
}
static void vector_fixed_vector(
const PODArray<UInt8> & cond,
const ColumnFixedString::Chars_t & a_data, const size_t a_N,
const ColumnString::Chars_t & b_data, const ColumnString::Offsets_t & b_offsets,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
vector_vector_fixed_impl<true>(cond, b_data, b_offsets, a_data, a_N, c_data, c_offsets);
}
template <bool negative>
static void vector_constant_impl(
const PODArray<UInt8> & cond,
const ColumnString::Chars_t & a_data, const ColumnString::Offsets_t & a_offsets,
const String & b,
@ -185,7 +264,7 @@ struct StringIfImpl
for (size_t i = 0; i < size; ++i)
{
if (cond[i])
if (negative != cond[i])
{
size_t size_to_write = a_offsets[i] - a_prev_offset;
c_data.resize(c_data.size() + size_to_write);
@ -206,42 +285,77 @@ struct StringIfImpl
}
}
static void vector_constant(
const PODArray<UInt8> & cond,
const ColumnString::Chars_t & a_data, const ColumnString::Offsets_t & a_offsets,
const String & b,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
return vector_constant_impl<false>(cond, a_data, a_offsets, b, c_data, c_offsets);
}
static void constant_vector(
const PODArray<UInt8> & cond,
const String & a,
const ColumnString::Chars_t & b_data, const ColumnString::Offsets_t & b_offsets,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
return vector_constant_impl<true>(cond, b_data, b_offsets, a, c_data, c_offsets);
}
template <bool negative>
static void vector_fixed_constant_impl(
const PODArray<UInt8> & cond,
const ColumnFixedString::Chars_t & a_data, const size_t a_N,
const String & b,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
size_t size = cond.size();
c_offsets.resize(size);
c_data.reserve(b_data.size());
c_data.reserve(a_data.size());
ColumnString::Offset_t b_prev_offset = 0;
ColumnString::Offset_t c_prev_offset = 0;
for (size_t i = 0; i < size; ++i)
{
if (cond[i])
if (negative != cond[i])
{
size_t size_to_write = a.size() + 1;
c_data.resize(c_data.size() + size_to_write);
memcpy(&c_data[c_prev_offset], a.data(), size_to_write);
c_prev_offset += size_to_write;
size_t size_to_write = a_N;
c_data.resize(c_data.size() + size_to_write + 1);
memcpy(&c_data[c_prev_offset], &a_data[i * a_N], size_to_write);
c_data.back() = 0;
c_prev_offset += size_to_write + 1;
c_offsets[i] = c_prev_offset;
}
else
{
size_t size_to_write = b_offsets[i] - b_prev_offset;
size_t size_to_write = b.size() + 1;
c_data.resize(c_data.size() + size_to_write);
memcpy(&c_data[c_prev_offset], &b_data[b_prev_offset], size_to_write);
memcpy(&c_data[c_prev_offset], b.data(), size_to_write);
c_prev_offset += size_to_write;
c_offsets[i] = c_prev_offset;
}
b_prev_offset = b_offsets[i];
}
}
static void vector_fixed_constant(
const PODArray<UInt8> & cond,
const ColumnFixedString::Chars_t & a_data, const size_t N,
const String & b,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
vector_fixed_constant_impl<false>(cond, a_data, N, b, c_data, c_offsets);
}
static void constant_vector_fixed(
const PODArray<UInt8> & cond,
const String & a,
const ColumnFixedString::Chars_t & b_data, const size_t N,
ColumnString::Chars_t & c_data, ColumnString::Offsets_t & c_offsets)
{
vector_fixed_constant_impl<true>(cond, b_data, N, a, c_data, c_offsets);
}
static void constant_constant(
const PODArray<UInt8> & cond,
const String & a, const String & b,
@ -495,6 +609,7 @@ public:
/** Реализация для массивов строк.
* NOTE: Код слишком сложный, потому что он работает в внутренностями массивов строк.
* NOTE: Массивы из FixedString не поддерживаются.
*/
struct StringArrayIfImpl
{
@ -1013,43 +1128,94 @@ private:
const ColumnString * col_then = typeid_cast<const ColumnString *>(col_then_untyped);
const ColumnString * col_else = typeid_cast<const ColumnString *>(col_else_untyped);
const ColumnFixedString * col_then_fixed = typeid_cast<const ColumnFixedString *>(col_then_untyped);
const ColumnFixedString * col_else_fixed = typeid_cast<const ColumnFixedString *>(col_else_untyped);
const ColumnConstString * col_then_const = typeid_cast<const ColumnConstString *>(col_then_untyped);
const ColumnConstString * col_else_const = typeid_cast<const ColumnConstString *>(col_else_untyped);
if ((col_then || col_then_const) && (col_else || col_else_const))
if ((col_then || col_then_const || col_then_fixed) && (col_else || col_else_const || col_else_fixed))
{
ColumnString * col_res = new ColumnString;
block.getByPosition(result).column = col_res;
if (col_then_fixed && col_else_fixed)
{
/// Результат - FixedString.
ColumnString::Chars_t & res_vec = col_res->getChars();
ColumnString::Offsets_t & res_offsets = col_res->getOffsets();
if (col_then_fixed->getN() != col_else_fixed->getN())
throw Exception("FixedString columns as 'then' and 'else' arguments of function 'if' has different sizes", ErrorCodes::ILLEGAL_COLUMN);
if (col_then && col_else)
StringIfImpl::vector_vector(
size_t N = col_then_fixed->getN();
ColumnFixedString * col_res = new ColumnFixedString(N);
block.getByPosition(result).column = col_res;
ColumnFixedString::Chars_t & res_vec = col_res->getChars();
StringIfImpl::vector_fixed_vector_fixed(
cond_col->getData(),
col_then->getChars(), col_then->getOffsets(),
col_else->getChars(), col_else->getOffsets(),
res_vec, res_offsets);
else if (col_then && col_else_const)
StringIfImpl::vector_constant(
cond_col->getData(),
col_then->getChars(), col_then->getOffsets(),
col_else_const->getData(),
res_vec, res_offsets);
else if (col_then_const && col_else)
StringIfImpl::constant_vector(
cond_col->getData(),
col_then_const->getData(),
col_else->getChars(), col_else->getOffsets(),
res_vec, res_offsets);
else if (col_then_const && col_else_const)
StringIfImpl::constant_constant(
cond_col->getData(),
col_then_const->getData(),
col_else_const->getData(),
res_vec, res_offsets);
col_then_fixed->getChars(),
col_else_fixed->getChars(),
N,
res_vec);
}
else
return false;
{
/// Результат - String.
ColumnString * col_res = new ColumnString;
block.getByPosition(result).column = col_res;
ColumnString::Chars_t & res_vec = col_res->getChars();
ColumnString::Offsets_t & res_offsets = col_res->getOffsets();
if (col_then && col_else)
StringIfImpl::vector_vector(
cond_col->getData(),
col_then->getChars(), col_then->getOffsets(),
col_else->getChars(), col_else->getOffsets(),
res_vec, res_offsets);
else if (col_then && col_else_const)
StringIfImpl::vector_constant(
cond_col->getData(),
col_then->getChars(), col_then->getOffsets(),
col_else_const->getData(),
res_vec, res_offsets);
else if (col_then_const && col_else)
StringIfImpl::constant_vector(
cond_col->getData(),
col_then_const->getData(),
col_else->getChars(), col_else->getOffsets(),
res_vec, res_offsets);
else if (col_then_const && col_else_const)
StringIfImpl::constant_constant(
cond_col->getData(),
col_then_const->getData(),
col_else_const->getData(),
res_vec, res_offsets);
else if (col_then && col_else_fixed)
StringIfImpl::vector_vector_fixed(
cond_col->getData(),
col_then->getChars(), col_then->getOffsets(),
col_else_fixed->getChars(), col_else_fixed->getN(),
res_vec, res_offsets);
else if (col_then_fixed && col_else)
StringIfImpl::vector_fixed_vector(
cond_col->getData(),
col_then_fixed->getChars(), col_then_fixed->getN(),
col_else->getChars(), col_else->getOffsets(),
res_vec, res_offsets);
else if (col_then_const && col_else_fixed)
StringIfImpl::constant_vector_fixed(
cond_col->getData(),
col_then_const->getData(),
col_else_fixed->getChars(), col_else_fixed->getN(),
res_vec, res_offsets);
else if (col_then_fixed && col_else_const)
StringIfImpl::vector_fixed_constant(
cond_col->getData(),
col_then_fixed->getChars(), col_then_fixed->getN(),
col_else_const->getData(),
res_vec, res_offsets);
else
return false;
}
return true;
}
@ -1151,9 +1317,29 @@ public:
}
else if (arguments[1]->getName() != arguments[2]->getName())
{
throw Exception("Incompatible second and third arguments for function " + getName() + ": "
+ arguments[1]->getName() + " and " + arguments[2]->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const DataTypeString * type_string1 = typeid_cast<const DataTypeString *>(arguments[1].get());
const DataTypeString * type_string2 = typeid_cast<const DataTypeString *>(arguments[2].get());
const DataTypeFixedString * type_fixed_string1 = typeid_cast<const DataTypeFixedString *>(arguments[1].get());
const DataTypeFixedString * type_fixed_string2 = typeid_cast<const DataTypeFixedString *>(arguments[2].get());
if (type_fixed_string1 && type_fixed_string2)
{
if (type_fixed_string1->getN() != type_fixed_string2->getN())
throw Exception("FixedString types as 'then' and 'else' arguments of function 'if' has different sizes",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return new DataTypeFixedString(type_fixed_string1->getN());
}
else if ((type_string1 || type_fixed_string1) && (type_string2 || type_fixed_string2))
{
return new DataTypeString;
}
throw Exception{
"Incompatible second and third arguments for function " + getName() + ": " +
arguments[1]->getName() + " and " + arguments[2]->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT
};
}
return arguments[1];

100
dbms/include/DB/Functions/FunctionsHashing.h
View File

@ -3,6 +3,8 @@
#include <openssl/md5.h>
#include <openssl/sha.h>
#include <city.h>
#include <farmhash.h>
#include <metrohash.h>
#include <Poco/ByteOrder.h>
@ -397,11 +399,12 @@ UInt64 toInteger<Float64>(Float64 x)
}
class FunctionCityHash64 : public IFunction
template <typename Impl>
class FunctionNeighbourhoodHash64 : public IFunction
{
public:
static constexpr auto name = "cityHash64";
static IFunction * create(const Context & context) { return new FunctionCityHash64; };
static constexpr auto name = Impl::name;
static IFunction * create(const Context & context) { return new FunctionNeighbourhoodHash64; };
private:
template <typename FromType, bool first>
@ -417,12 +420,12 @@ private:
if (first)
vec_to[i] = h;
else
vec_to[i] = Hash128to64(uint128(vec_to[i], h));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], h));
}
}
else if (const ColumnConst<FromType> * col_from = typeid_cast<const ColumnConst<FromType> *>(column))
{
UInt64 hash = IntHash64Impl::apply(toInteger(col_from->getData()));
const UInt64 hash = IntHash64Impl::apply(toInteger(col_from->getData()));
size_t size = vec_to.size();
if (first)
{
@ -431,7 +434,7 @@ private:
else
{
for (size_t i = 0; i < size; ++i)
vec_to[i] = Hash128to64(uint128(vec_to[i], hash));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], hash));
}
}
else
@ -451,13 +454,13 @@ private:
for (size_t i = 0; i < size; ++i)
{
UInt64 h = CityHash64(
const UInt64 h = Impl::Hash64(
reinterpret_cast<const char *>(&data[i == 0 ? 0 : offsets[i - 1]]),
i == 0 ? offsets[i] - 1 : (offsets[i] - 1 - offsets[i - 1]));
if (first)
vec_to[i] = h;
else
vec_to[i] = Hash128to64(uint128(vec_to[i], h));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], h));
}
}
else if (const ColumnFixedString * col_from = typeid_cast<const ColumnFixedString *>(column))
@ -467,17 +470,17 @@ private:
size_t size = data.size() / n;
for (size_t i = 0; i < size; ++i)
{
UInt64 h = CityHash64(reinterpret_cast<const char *>(&data[i * n]), n);
const UInt64 h = Impl::Hash64(reinterpret_cast<const char *>(&data[i * n]), n);
if (first)
vec_to[i] = h;
else
vec_to[i] = Hash128to64(uint128(vec_to[i], h));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], h));
}
}
else if (const ColumnConstString * col_from = typeid_cast<const ColumnConstString *>(column))
{
UInt64 hash = CityHash64(col_from->getData().data(), col_from->getData().size());
size_t size = vec_to.size();
const UInt64 hash = Impl::Hash64(col_from->getData().data(), col_from->getData().size());
const size_t size = vec_to.size();
if (first)
{
vec_to.assign(size, hash);
@ -486,7 +489,7 @@ private:
{
for (size_t i = 0; i < size; ++i)
{
vec_to[i] = Hash128to64(uint128(vec_to[i], hash));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], hash));
}
}
}
@ -505,26 +508,26 @@ private:
{
const IColumn * nested_column = &col_from->getData();
const ColumnArray::Offsets_t & offsets = col_from->getOffsets();
size_t nested_size = nested_column->size();
const size_t nested_size = nested_column->size();
ColumnUInt64::Container_t vec_temp(nested_size);
executeAny<true>(nested_type, nested_column, vec_temp);
size_t size = offsets.size();
const size_t size = offsets.size();
for (size_t i = 0; i < size; ++i)
{
size_t begin = i == 0 ? 0 : offsets[i - 1];
size_t end = offsets[i];
const size_t begin = i == 0 ? 0 : offsets[i - 1];
const size_t end = offsets[i];
UInt64 h = IntHash64Impl::apply(end - begin);
if (first)
vec_to[i] = h;
else
vec_to[i] = Hash128to64(uint128(vec_to[i], h));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], h));
for (size_t j = begin; j < end; ++j)
vec_to[i] = Hash128to64(uint128(vec_to[i], vec_temp[j]));
vec_to[i] = Impl::Hash128to64(typename Impl::uint128_t(vec_to[i], vec_temp[j]));
}
}
else if (const ColumnConstArray * col_from = typeid_cast<const ColumnConstArray *>(column))
@ -816,18 +819,57 @@ private:
struct NameHalfMD5 { static constexpr auto name = "halfMD5"; };
struct NameSipHash64 { static constexpr auto name = "sipHash64"; };
struct NameCityHash64 { static constexpr auto name = "cityHash64"; };
struct NameIntHash32 { static constexpr auto name = "intHash32"; };
struct NameIntHash64 { static constexpr auto name = "intHash64"; };
typedef FunctionStringHash64<HalfMD5Impl, NameHalfMD5> FunctionHalfMD5;
typedef FunctionStringHash64<SipHash64Impl, NameSipHash64> FunctionSipHash64;
typedef FunctionIntHash<IntHash32Impl, NameIntHash32> FunctionIntHash32;
typedef FunctionIntHash<IntHash64Impl, NameIntHash64> FunctionIntHash64;
typedef FunctionStringHashFixedString<MD5Impl> FunctionMD5;
typedef FunctionStringHashFixedString<SHA1Impl> FunctionSHA1;
typedef FunctionStringHashFixedString<SHA224Impl> FunctionSHA224;
typedef FunctionStringHashFixedString<SHA256Impl> FunctionSHA256;
typedef FunctionStringHashFixedString<SipHash128Impl> FunctionSipHash128;
struct ImplCityHash64
{
static constexpr auto name = "cityHash64";
using uint128_t = uint128;
static auto Hash128to64(const uint128_t & x) { return ::Hash128to64(x); }
static auto Hash64(const char * const s, const std::size_t len) { return CityHash64(s, len); }
};
struct ImplFarmHash64
{
static constexpr auto name = "farmHash64";
using uint128_t = farmhash::uint128_t;
static auto Hash128to64(const uint128_t & x) { return farmhash::Hash128to64(x); }
static auto Hash64(const char * const s, const std::size_t len) { return farmhash::Hash64(s, len); }
};
struct ImplMetroHash64
{
static constexpr auto name = "metroHash64";
using uint128_t = uint128;
static auto Hash128to64(const uint128_t & x) { return ::Hash128to64(x); }
static auto Hash64(const char * const s, const std::size_t len)
{
union {
std::uint64_t u64;
std::uint8_t u8[sizeof(u64)];
};
metrohash64_1(reinterpret_cast<const std::uint8_t *>(s), len, 0, u8);
return u64;
}
};
using FunctionHalfMD5 = FunctionStringHash64<HalfMD5Impl, NameHalfMD5>;
using FunctionSipHash64 = FunctionStringHash64<SipHash64Impl, NameSipHash64>;
using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
using FunctionMD5 = FunctionStringHashFixedString<MD5Impl>;
using FunctionSHA1 = FunctionStringHashFixedString<SHA1Impl>;
using FunctionSHA224 = FunctionStringHashFixedString<SHA224Impl>;
using FunctionSHA256 = FunctionStringHashFixedString<SHA256Impl>;
using FunctionSipHash128 = FunctionStringHashFixedString<SipHash128Impl>;
using FunctionCityHash64 = FunctionNeighbourhoodHash64<ImplCityHash64>;
using FunctionFarmHash64 = FunctionNeighbourhoodHash64<ImplFarmHash64>;
using FunctionMetroHash64 = FunctionNeighbourhoodHash64<ImplMetroHash64>;
}

16
dbms/include/DB/Interpreters/Aggregator.h
View File

@ -374,8 +374,20 @@ struct AggregationMethodConcat
/// См. функцию extractKeysAndPlaceInPoolContiguous.
const StringRef * key_refs = reinterpret_cast<const StringRef *>(value.first.data + value.first.size);
for (size_t i = 0; i < keys_size; ++i)
key_columns[i]->insertDataWithTerminatingZero(key_refs[i].data, key_refs[i].size);
if (unlikely(0 == value.first.size))
{
/** Исправление, если все ключи - пустые массивы. Для них в хэш-таблицу записывается StringRef нулевой длины, но с ненулевым указателем.
* Но при вставке в хэш-таблицу, такой StringRef оказывается равен другому ключу нулевой длины,
* у которого указатель на данные может быть любым мусором и использовать его нельзя.
*/
for (size_t i = 0; i < keys_size; ++i)
key_columns[i]->insertDefault();
}
else
{
for (size_t i = 0; i < keys_size; ++i)
key_columns[i]->insertDataWithTerminatingZero(key_refs[i].data, key_refs[i].size);
}
}
};

3
dbms/src/DataStreams/FormatFactory.cpp
View File

@ -17,6 +17,7 @@
#include <DB/DataStreams/BlockOutputStreamFromRowOutputStream.h>
#include <DB/DataStreams/JSONRowOutputStream.h>
#include <DB/DataStreams/JSONCompactRowOutputStream.h>
#include <DB/DataStreams/TSKVRowOutputStream.h>
#include <DB/DataStreams/PrettyCompactMonoBlockOutputStream.h>
#include <DB/DataStreams/FormatFactory.h>
@ -83,6 +84,8 @@ BlockOutputStreamPtr FormatFactory::getOutput(const String & name, WriteBuffer &
return new BlockOutputStreamFromRowOutputStream(new JSONRowOutputStream(buf, sample));
else if (name == "JSONCompact")
return new BlockOutputStreamFromRowOutputStream(new JSONCompactRowOutputStream(buf, sample));
else if (name == "TSKV")
return new BlockOutputStreamFromRowOutputStream(new TSKVRowOutputStream(buf, sample));
else if (name == "Null")
return new NullBlockOutputStream;
else if (name == "PrettyCompactMonoBlock")

37
dbms/src/DataStreams/TSKVRowOutputStream.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <DB/IO/WriteHelpers.h>
#include <DB/DataStreams/TSKVRowOutputStream.h>
namespace DB
{
using Poco::SharedPtr;
TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_)
: TabSeparatedRowOutputStream(ostr_, sample_)
{
NamesAndTypesList columns(sample_.getColumnsList());
fields.assign(columns.begin(), columns.end());
for (auto & field : fields)
{
String escaped_field_name;
{
WriteBufferFromString wb(escaped_field_name);
writeAnyEscapedString<'='>(field.name.data(), field.name.data() + field.name.size(), wb);
}
field.name = escaped_field_name;
}
}
void TSKVRowOutputStream::writeField(const Field & field)
{
writeString(fields[field_number].name, ostr);
writeCString("=", ostr);
data_types[field_number]->serializeTextEscaped(field, ostr);
++field_number;
}
}

2
dbms/src/Functions/FunctionsHashing.cpp
View File

@ -15,6 +15,8 @@ void registerFunctionsHashing(FunctionFactory & factory)
factory.registerFunction<FunctionSipHash64>();
factory.registerFunction<FunctionSipHash128>();
factory.registerFunction<FunctionCityHash64>();
factory.registerFunction<FunctionFarmHash64>();
factory.registerFunction<FunctionMetroHash64>();
factory.registerFunction<FunctionIntHash32>();
factory.registerFunction<FunctionIntHash64>();
factory.registerFunction<FunctionURLHash>();

2
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -2271,7 +2271,7 @@ void StorageReplicatedMergeTree::dropPartition(const Field & field, bool detach,
/// TODO: Делать запрос в лидера по TCP.
if (!is_leader_node)
throw Exception("DROP PARTITION can only be done on leader replica.", ErrorCodes::NOT_LEADER);
throw Exception(String(detach ? "DETACH" : "DROP") + " PARTITION can only be done on leader replica.", ErrorCodes::NOT_LEADER);
/** Пропустим один номер в block_numbers для удаляемого месяца, и будем удалять только куски до этого номера.
* Это запретит мерджи удаляемых кусков с новыми вставляемыми данными.

90
dbms/tests/queries/0_stateless/00197_if_fixed_string.reference Normal file
View File

@ -0,0 +1,90 @@
0
1
-2
3
-4
5
-6
7
-8
9
0\0
1\0
-2
3\0
-4
5\0
-6
7\0
-8
9\0
0
1\0
-2
3\0
-4
5\0
-6
7\0
-8
9\0
0\0
1
-2
3
-4
5
-6
7
-8
9
Hello
1
Hello
3
Hello
5
Hello
7
Hello
9
0
Hello
-2
Hello
-4
Hello
-6
Hello
-8
Hello
Goodbye
Hello
Goodbye
Hello
Goodbye
Hello
Goodbye
Hello
Goodbye
Hello
Hello
1\0
Hello
3\0
Hello
5\0
Hello
7\0
Hello
9\0
0\0
Hello
-2
Hello
-4
Hello
-6
Hello
-8
Hello

9
dbms/tests/queries/0_stateless/00197_if_fixed_string.sql Normal file
View File

@ -0,0 +1,9 @@
SELECT number % 2 ? toString(number) : toString(-number) FROM system.numbers LIMIT 10;
SELECT number % 2 ? toFixedString(toString(number), 2) : toFixedString(toString(-number), 2) FROM system.numbers LIMIT 10;
SELECT number % 2 ? toFixedString(toString(number), 2) : toString(-number) FROM system.numbers LIMIT 10;
SELECT number % 2 ? toString(number) : toFixedString(toString(-number), 2) FROM system.numbers LIMIT 10;
SELECT number % 2 ? toString(number) : 'Hello' FROM system.numbers LIMIT 10;
SELECT number % 2 ? 'Hello' : toString(-number) FROM system.numbers LIMIT 10;
SELECT number % 2 ? 'Hello' : 'Goodbye' FROM system.numbers LIMIT 10;
SELECT number % 2 ? toFixedString(toString(number), 2) : 'Hello' FROM system.numbers LIMIT 10;
SELECT number % 2 ? 'Hello' : toFixedString(toString(-number), 2) FROM system.numbers LIMIT 10;

23
dbms/tests/queries/0_stateless/00198_group_by_empty_arrays.reference Normal file
View File

@ -0,0 +1,23 @@
[] 5
[0] 1
[0,1,2] 1
[0,1,2,3,4] 1
[0,1,2,3,4,5,6] 1
[0,1,2,3,4,5,6,7,8] 1
[] [] 4
[] [0,1,2,3] 1
[] [0,1,2,3,4,5] 1
[] [0,1,2,3,4,5,6,7,8,9] 1
[] [0,1,2,3,4,5,6,7,8,9,10,11] 1
[] [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] 1
[] [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] 1
[0] [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18] 1
[0,1,2] [] 1
[0,1,2,3,4] [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14] 1
[0,1,2,3,4,5,6] [0,1,2,3,4,5,6,7,8,9,10,11,12] 1
[0,1,2,3,4,5,6,7,8] [] 1
[0,1,2,3,4,5,6,7,8,9,10] [0,1,2,3,4,5,6,7,8] 1
[0,1,2,3,4,5,6,7,8,9,10,11,12] [0,1,2,3,4,5,6] 1
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14] [] 1
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16] [0,1,2] 1
[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18] [0] 1

2
dbms/tests/queries/0_stateless/00198_group_by_empty_arrays.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT range(x) AS k, count() FROM (SELECT number % 2 ? number : 0 AS x FROM system.numbers LIMIT 10) GROUP BY k ORDER BY k;
SELECT range(x) AS k1, range(y) AS k2, count() FROM (SELECT number % 2 ? number : 0 AS x, number % 3 ? toUInt64(20 - number) : 0 AS y FROM system.numbers LIMIT 20) GROUP BY k1, k2 ORDER BY k1, k2;

62
dbms/tests/queries/0_stateless/00199_ternary_operator_type_check.reference Normal file
View File

@ -0,0 +1,62 @@
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1

80
dbms/tests/queries/0_stateless/00199_ternary_operator_type_check.sql Normal file
View File

@ -0,0 +1,80 @@
select (1 ? ('abc' as s) : 'def') = s;
select (1 ? toFixedString('abc' as s, 3) : 'def') = s;
select (1 ? toFixedString('abc' as s, 3) : toFixedString('def', 3)) = s;
select (1 ? ('abc' as s) : toFixedString('def', 3)) = s;
select (1 ? (today() as t) : yesterday()) = t;
select (1 ? (now() as n) : now() - 1) = n;
select (1 ? (toUInt8(0) as i) : toUInt8(1)) = i;
select (1 ? (toUInt16(0) as i) : toUInt8(1)) = i;
select (1 ? (toUInt32(0) as i) : toUInt8(1)) = i;
select (1 ? (toUInt64(0) as i) : toUInt8(1)) = i;
select (1 ? (toInt8(0) as i) : toUInt8(1)) = i;
select (1 ? (toInt16(0) as i) : toUInt8(1)) = i;
select (1 ? (toInt32(0) as i) : toUInt8(1)) = i;
select (1 ? (toInt64(0) as i) : toUInt8(1)) = i;
select (1 ? (toUInt8(0) as i) : toUInt16(1)) = i;
select (1 ? (toUInt16(0) as i) : toUInt16(1)) = i;
select (1 ? (toUInt32(0) as i) : toUInt16(1)) = i;
select (1 ? (toUInt64(0) as i) : toUInt16(1)) = i;
select (1 ? (toInt8(0) as i) : toUInt16(1)) = i;
select (1 ? (toInt16(0) as i) : toUInt16(1)) = i;
select (1 ? (toInt32(0) as i) : toUInt16(1)) = i;
select (1 ? (toInt64(0) as i) : toUInt16(1)) = i;
select (1 ? (toUInt8(0) as i) : toUInt32(1)) = i;
select (1 ? (toUInt16(0) as i) : toUInt32(1)) = i;
select (1 ? (toUInt32(0) as i) : toUInt32(1)) = i;
select (1 ? (toUInt64(0) as i) : toUInt32(1)) = i;
select (1 ? (toInt8(0) as i) : toUInt32(1)) = i;
select (1 ? (toInt16(0) as i) : toUInt32(1)) = i;
select (1 ? (toInt32(0) as i) : toUInt32(1)) = i;
select (1 ? (toInt64(0) as i) : toUInt32(1)) = i;
select (1 ? (toUInt8(0) as i) : toUInt64(1)) = i;
select (1 ? (toUInt16(0) as i) : toUInt64(1)) = i;
select (1 ? (toUInt32(0) as i) : toUInt64(1)) = i;
select (1 ? (toUInt64(0) as i) : toUInt64(1)) = i;
--select (1 ? (toInt8(0) as i) : toUInt64(1)) = i;
--select (1 ? (toInt16(0) as i) : toUInt64(1)) = i;
--select (1 ? (toInt32(0) as i) : toUInt64(1)) = i;
--select (1 ? (toInt64(0) as i) : toUInt64(1)) = i;
select (1 ? (toUInt8(0) as i) : toInt8(1)) = i;
select (1 ? (toUInt16(0) as i) : toInt8(1)) = i;
select (1 ? (toUInt32(0) as i) : toInt8(1)) = i;
--select (1 ? (toUInt64(0) as i) : toInt8(1)) = i;
select (1 ? (toInt8(0) as i) : toInt8(1)) = i;
select (1 ? (toInt16(0) as i) : toInt8(1)) = i;
select (1 ? (toInt32(0) as i) : toInt8(1)) = i;
select (1 ? (toInt64(0) as i) : toInt8(1)) = i;
select (1 ? (toUInt8(0) as i) : toInt16(1)) = i;
select (1 ? (toUInt16(0) as i) : toInt16(1)) = i;
select (1 ? (toUInt32(0) as i) : toInt16(1)) = i;
--select (1 ? (toUInt64(0) as i) : toInt16(1)) = i;
select (1 ? (toInt8(0) as i) : toInt16(1)) = i;
select (1 ? (toInt16(0) as i) : toInt16(1)) = i;
select (1 ? (toInt32(0) as i) : toInt16(1)) = i;
select (1 ? (toInt64(0) as i) : toInt16(1)) = i;
select (1 ? (toUInt8(0) as i) : toInt32(1)) = i;
select (1 ? (toUInt16(0) as i) : toInt32(1)) = i;
select (1 ? (toUInt32(0) as i) : toInt32(1)) = i;
--select (1 ? (toUInt64(0) as i) : toInt32(1)) = i;
select (1 ? (toInt8(0) as i) : toInt32(1)) = i;
select (1 ? (toInt16(0) as i) : toInt32(1)) = i;
select (1 ? (toInt32(0) as i) : toInt32(1)) = i;
select (1 ? (toInt64(0) as i) : toInt32(1)) = i;
select (1 ? (toUInt8(0) as i) : toInt64(1)) = i;
select (1 ? (toUInt16(0) as i) : toInt64(1)) = i;
select (1 ? (toUInt32(0) as i) : toInt64(1)) = i;
--select (1 ? (toUInt64(0) as i) : toInt64(1)) = i;
select (1 ? (toInt8(0) as i) : toInt64(1)) = i;
select (1 ? (toInt16(0) as i) : toInt64(1)) = i;
select (1 ? (toInt32(0) as i) : toInt64(1)) = i;
select (1 ? (toInt64(0) as i) : toInt64(1)) = i;

8
libs/libfarmhash/README Normal file
View File

@ -0,0 +1,8 @@
Original URL: https://code.google.com/p/farmhash/
Created from version:
commit afdb2e459d030afd1f3b6116545397509326385e
Author: gpike@google.com <gpike@google.com@34f43262-9e73-f86a-efff-89e1528fadaf>
Date: Sun Mar 1 20:04:23 2015 +0000
Various additions, updates, and fixes for FarmHash 1.1.

11854
libs/libfarmhash/farmhash.cc Normal file
View File

File diff suppressed because it is too large Load Diff

290
libs/libfarmhash/farmhash.h Normal file
View File

@ -0,0 +1,290 @@
// Copyright (c) 2014 Google, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// FarmHash, by Geoff Pike
//
// http://code.google.com/p/farmhash/
//
// This file provides a few functions for hashing strings and other
// data. All of them are high-quality functions in the sense that
// they do well on standard tests such as Austin Appleby's SMHasher.
// They're also fast. FarmHash is the successor to CityHash.
//
// Functions in the FarmHash family are not suitable for cryptography.
//
// WARNING: This code has been only lightly tested on big-endian platforms!
// It is known to work well on little-endian platforms that have a small penalty
// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs.
// It should work on all 32-bit and 64-bit platforms that allow unaligned reads;
// bug reports are welcome.
//
// By the way, for some hash functions, given strings a and b, the hash
// of a+b is easily derived from the hashes of a and b. This property
// doesn't hold for any hash functions in this file.
#ifndef FARM_HASH_H_
#define FARM_HASH_H_
#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h> // for memcpy and memset
#include <utility>
#ifndef NAMESPACE_FOR_HASH_FUNCTIONS
#define NAMESPACE_FOR_HASH_FUNCTIONS farmhash
#endif
namespace NAMESPACE_FOR_HASH_FUNCTIONS {
#if defined(FARMHASH_UINT128_T_DEFINED)
inline uint64_t Uint128Low64(const uint128_t x) {
return static_cast<uint64_t>(x);
}
inline uint64_t Uint128High64(const uint128_t x) {
return static_cast<uint64_t>(x >> 64);
}
inline uint128_t Uint128(uint64_t lo, uint64_t hi) {
return lo + (((uint128_t)hi) << 64);
}
#else
typedef std::pair<uint64_t, uint64_t> uint128_t;
inline uint64_t Uint128Low64(const uint128_t x) { return x.first; }
inline uint64_t Uint128High64(const uint128_t x) { return x.second; }
inline uint128_t Uint128(uint64_t lo, uint64_t hi) { return uint128_t(lo, hi); }
#endif
// BASIC STRING HASHING
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
size_t Hash(const char* s, size_t len);
// Hash function for a byte array. Most useful in 32-bit binaries.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint32_t Hash32(const char* s, size_t len);
// Hash function for a byte array. For convenience, a 32-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint32_t Hash32WithSeed(const char* s, size_t len, uint32_t seed);
// Hash 128 input bits down to 64 bits of output.
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint64_t Hash64(const char* s, size_t len);
// Hash function for a byte array. For convenience, a 64-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint64_t Hash64WithSeed(const char* s, size_t len, uint64_t seed);
// Hash function for a byte array. For convenience, two seeds are also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint64_t Hash64WithSeeds(const char* s, size_t len,
uint64_t seed0, uint64_t seed1);
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint128_t Hash128(const char* s, size_t len);
// Hash function for a byte array. For convenience, a 128-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
uint128_t Hash128WithSeed(const char* s, size_t len, uint128_t seed);
// BASIC NON-STRING HASHING
// This is intended to be a reasonably good hash function.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
inline uint64_t Hash128to64(uint128_t x) {
// Murmur-inspired hashing.
const uint64_t kMul = 0x9ddfea08eb382d69ULL;
uint64_t a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul;
a ^= (a >> 47);
uint64_t b = (Uint128High64(x) ^ a) * kMul;
b ^= (b >> 47);
b *= kMul;
return b;
}
// FINGERPRINTING (i.e., good, portable, forever-fixed hash functions)
// Fingerprint function for a byte array. Most useful in 32-bit binaries.
uint32_t Fingerprint32(const char* s, size_t len);
// Fingerprint function for a byte array.
uint64_t Fingerprint64(const char* s, size_t len);
// Fingerprint function for a byte array.
uint128_t Fingerprint128(const char* s, size_t len);
// This is intended to be a good fingerprinting primitive.
// See below for more overloads.
inline uint64_t Fingerprint(uint128_t x) {
// Murmur-inspired hashing.
const uint64_t kMul = 0x9ddfea08eb382d69ULL;
uint64_t a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul;
a ^= (a >> 47);
uint64_t b = (Uint128High64(x) ^ a) * kMul;
b ^= (b >> 44);
b *= kMul;
b ^= (b >> 41);
b *= kMul;
return b;
}
// This is intended to be a good fingerprinting primitive.
inline uint64_t Fingerprint(uint64_t x) {
// Murmur-inspired hashing.
const uint64_t kMul = 0x9ddfea08eb382d69ULL;
uint64_t b = x * kMul;
b ^= (b >> 44);
b *= kMul;
b ^= (b >> 41);
b *= kMul;
return b;
}
#ifndef FARMHASH_NO_CXX_STRING
// Convenience functions to hash or fingerprint C++ strings.
// These require that Str::data() return a pointer to the first char
// (as a const char*) and that Str::length() return the string's length;
// they work with std::string, for example.
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline size_t Hash(const Str& s) {
assert(sizeof(s[0]) == 1);
return Hash(s.data(), s.length());
}
// Hash function for a byte array. Most useful in 32-bit binaries.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint32_t Hash32(const Str& s) {
assert(sizeof(s[0]) == 1);
return Hash32(s.data(), s.length());
}
// Hash function for a byte array. For convenience, a 32-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint32_t Hash32WithSeed(const Str& s, uint32_t seed) {
assert(sizeof(s[0]) == 1);
return Hash32WithSeed(s.data(), s.length(), seed);
}
// Hash 128 input bits down to 64 bits of output.
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint64_t Hash64(const Str& s) {
assert(sizeof(s[0]) == 1);
return Hash64(s.data(), s.length());
}
// Hash function for a byte array. For convenience, a 64-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint64_t Hash64WithSeed(const Str& s, uint64_t seed) {
assert(sizeof(s[0]) == 1);
return Hash64WithSeed(s.data(), s.length(), seed);
}
// Hash function for a byte array. For convenience, two seeds are also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint64_t Hash64WithSeeds(const Str& s, uint64_t seed0, uint64_t seed1) {
assert(sizeof(s[0]) == 1);
return Hash64WithSeeds(s.data(), s.length(), seed0, seed1);
}
// Hash function for a byte array.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint128_t Hash128(const Str& s) {
assert(sizeof(s[0]) == 1);
return Hash128(s.data(), s.length());
}
// Hash function for a byte array. For convenience, a 128-bit seed is also
// hashed into the result.
// May change from time to time, may differ on different platforms, may differ
// depending on NDEBUG.
template <typename Str>
inline uint128_t Hash128WithSeed(const Str& s, uint128_t seed) {
assert(sizeof(s[0]) == 1);
return Hash128(s.data(), s.length(), seed);
}
// FINGERPRINTING (i.e., good, portable, forever-fixed hash functions)
// Fingerprint function for a byte array. Most useful in 32-bit binaries.
template <typename Str>
inline uint32_t Fingerprint32(const Str& s) {
assert(sizeof(s[0]) == 1);
return Fingerprint32(s.data(), s.length());
}
// Fingerprint 128 input bits down to 64 bits of output.
// Fingerprint function for a byte array.
template <typename Str>
inline uint64_t Fingerprint64(const Str& s) {
assert(sizeof(s[0]) == 1);
return Fingerprint64(s.data(), s.length());
}
// Fingerprint function for a byte array.
template <typename Str>
inline uint128_t Fingerprint128(const Str& s) {
assert(sizeof(s[0]) == 1);
return Fingerprint128(s.data(), s.length());
}
#endif
} // namespace NAMESPACE_FOR_HASH_FUNCTIONS
#endif // FARM_HASH_H_

22
libs/libmetrohash/LICENSE Normal file
View File

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 J. Andrew Rogers
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

24
libs/libmetrohash/README.md Normal file
View File

@ -0,0 +1,24 @@
## MetroHash: Faster, Better Hash Functions
MetroHash is a set of state-of-the-art hash functions for *non-cryptographic* use cases. They are notable for being algorithmically generated in addition to their exceptional performance. The set of published hash functions may be expanded in the future, having been selected from a very large set of hash functions that have been constructed this way.
* Fastest general-purpose functions for bulk hashing.
* Fastest general-purpose functions for small, variable length keys.
* Robust statistical bias profile, similar to the MD5 cryptographic hash.
* 64-bit, 128-bit, and 128-bit CRC variants currently available.
* Optimized for modern x86-64 microarchitectures.
* Elegant, compact, readable functions.
You can read more about the design and history [here](http://www.jandrewrogers.com/2015/05/27/metrohash/).
Six hash functions have been included in the initial release:
* 64-bit hash functions, "metrohash64_1" and "metrohash64_2"
* 128-bit hash functions, "metrohash128_1" and "metrohash128_2"
* 128-bit hash functions using CRC instructions, "metrohash128crc_1" and "metrohash128crc_2"
Hash functions in the same family are effectively statistically unique. In other words, if you need two hash functions for a bloom filter, you can use "metrohash64_1" and "metrohash64_2" in the same implementation without issue. An unbounded set of statistically unique functions can be generated in each family. The functions in this repo were generated specifically for public release.
The hash function generation software made no effort toward portability. While these hash functions should be easily portable to big-endian microarchitectures, they have not been tested on them and the performance optimization algorithms were not targeted at them. ARM64 microarchitectures might be a worthwhile hash function generation targets if I had the hardware.

7
libs/libmetrohash/VERSION Normal file
View File

@ -0,0 +1,7 @@
origin: git@github.com:jandrewrogers/MetroHash.git
commit d9dee18a54a8a6766e24c1950b814ac7ca9d1a89
Merge: 761e8a4 3d06b24
Author: J. Andrew Rogers <andrew@jarbox.org>
Date: Sat Jun 6 16:12:06 2015 -0700
modified README

73
libs/libmetrohash/src/metrohash.h Normal file
View File

@ -0,0 +1,73 @@
// metrohash.h
//
// The MIT License (MIT)
//
// Copyright (c) 2015 J. Andrew Rogers
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
#ifndef METROHASH_METROHASH_H
#define METROHASH_METROHASH_H
#include <stdint.h>
#include <string.h>
// MetroHash 64-bit hash functions
void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
// MetroHash 128-bit hash functions
void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
// MetroHash 128-bit hash functions using CRC instruction
void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
void metrohash128crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
/* rotate right idiom recognized by compiler*/
inline static uint64_t rotate_right(uint64_t v, unsigned k)
{
return (v >> k) | (v << (64 - k));
}
// unaligned reads, fast and safe on Nehalem and later microarchitectures
inline static uint64_t read_u64(const void * const ptr)
{
return static_cast<uint64_t>(*reinterpret_cast<const uint64_t*>(ptr));
}
inline static uint64_t read_u32(const void * const ptr)
{
return static_cast<uint64_t>(*reinterpret_cast<const uint32_t*>(ptr));
}
inline static uint64_t read_u16(const void * const ptr)
{
return static_cast<uint64_t>(*reinterpret_cast<const uint16_t*>(ptr));
}
inline static uint64_t read_u8 (const void * const ptr)
{
return static_cast<uint64_t>(*reinterpret_cast<const uint8_t *>(ptr));
}
#endif // #ifndef METROHASH_METROHASH_H

178
libs/libmetrohash/src/metrohash128.cpp Normal file
View File

@ -0,0 +1,178 @@
// metrohash128.cpp
//
// The MIT License (MIT)
//
// Copyright (c) 2015 J. Andrew Rogers
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
#include "metrohash.h"
void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xC83A91E1;
static const uint64_t k1 = 0x8648DBDB;
static const uint64_t k2 = 0x7BDEC03B;
static const uint64_t k3 = 0x2F5870A5;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t v[4];
v[0] = ((static_cast<uint64_t>(seed) - k0) * k3) + len;
v[1] = ((static_cast<uint64_t>(seed) + k1) * k2) + len;
if (len >= 32)
{
v[2] = ((static_cast<uint64_t>(seed) + k0) * k2) + len;
v[3] = ((static_cast<uint64_t>(seed) - k1) * k3) + len;
do
{
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 26) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 26) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 26) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 30) * k0;
}
if ((end - ptr) >= 16)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],33) * k3;
v[1] += read_u64(ptr) * k2; ptr += 8; v[1] = rotate_right(v[1],33) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 17) * k1;
v[1] ^= rotate_right((v[1] * k3) + v[0], 17) * k0;
}
if ((end - ptr) >= 8)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],33) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 20) * k1;
}
if ((end - ptr) >= 4)
{
v[1] += read_u32(ptr) * k2; ptr += 4; v[1] = rotate_right(v[1],33) * k3;
v[1] ^= rotate_right((v[1] * k3) + v[0], 18) * k0;
}
if ((end - ptr) >= 2)
{
v[0] += read_u16(ptr) * k2; ptr += 2; v[0] = rotate_right(v[0],33) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 24) * k1;
}
if ((end - ptr) >= 1)
{
v[1] += read_u8 (ptr) * k2; v[1] = rotate_right(v[1],33) * k3;
v[1] ^= rotate_right((v[1] * k3) + v[0], 24) * k0;
}
v[0] += rotate_right((v[0] * k0) + v[1], 13);
v[1] += rotate_right((v[1] * k1) + v[0], 37);
v[0] += rotate_right((v[0] * k2) + v[1], 13);
v[1] += rotate_right((v[1] * k3) + v[0], 37);
memcpy(out, v, 16);
}
void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xD6D018F5;
static const uint64_t k1 = 0xA2AA033B;
static const uint64_t k2 = 0x62992FC1;
static const uint64_t k3 = 0x30BC5B29;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t v[4];
v[0] = ((static_cast<uint64_t>(seed) - k0) * k3) + len;
v[1] = ((static_cast<uint64_t>(seed) + k1) * k2) + len;
if (len >= 32)
{
v[2] = ((static_cast<uint64_t>(seed) + k0) * k2) + len;
v[3] = ((static_cast<uint64_t>(seed) - k1) * k3) + len;
do
{
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 33) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 33) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 33) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 33) * k0;
}
if ((end - ptr) >= 16)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],29) * k3;
v[1] += read_u64(ptr) * k2; ptr += 8; v[1] = rotate_right(v[1],29) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 29) * k1;
v[1] ^= rotate_right((v[1] * k3) + v[0], 29) * k0;
}
if ((end - ptr) >= 8)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],29) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 29) * k1;
}
if ((end - ptr) >= 4)
{
v[1] += read_u32(ptr) * k2; ptr += 4; v[1] = rotate_right(v[1],29) * k3;
v[1] ^= rotate_right((v[1] * k3) + v[0], 25) * k0;
}
if ((end - ptr) >= 2)
{
v[0] += read_u16(ptr) * k2; ptr += 2; v[0] = rotate_right(v[0],29) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 30) * k1;
}
if ((end - ptr) >= 1)
{
v[1] += read_u8 (ptr) * k2; v[1] = rotate_right(v[1],29) * k3;
v[1] ^= rotate_right((v[1] * k3) + v[0], 18) * k0;
}
v[0] += rotate_right((v[0] * k0) + v[1], 33);
v[1] += rotate_right((v[1] * k1) + v[0], 33);
v[0] += rotate_right((v[0] * k2) + v[1], 33);
v[1] += rotate_right((v[1] * k3) + v[0], 33);
memcpy(out, v, 16);
}

180
libs/libmetrohash/src/metrohash128crc.cpp Normal file
View File

@ -0,0 +1,180 @@
// metrohash128crc.cpp
//
// The MIT License (MIT)
//
// Copyright (c) 2015 J. Andrew Rogers
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
#include "metrohash.h"
#include <nmmintrin.h>
void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xC83A91E1;
static const uint64_t k1 = 0x8648DBDB;
static const uint64_t k2 = 0x7BDEC03B;
static const uint64_t k3 = 0x2F5870A5;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t v[4];
v[0] = ((static_cast<uint64_t>(seed) - k0) * k3) + len;
v[1] = ((static_cast<uint64_t>(seed) + k1) * k2) + len;
if (len >= 32)
{
v[2] = ((static_cast<uint64_t>(seed) + k0) * k2) + len;
v[3] = ((static_cast<uint64_t>(seed) - k1) * k3) + len;
do
{
v[0] ^= _mm_crc32_u64(v[0], read_u64(ptr)); ptr += 8;
v[1] ^= _mm_crc32_u64(v[1], read_u64(ptr)); ptr += 8;
v[2] ^= _mm_crc32_u64(v[2], read_u64(ptr)); ptr += 8;
v[3] ^= _mm_crc32_u64(v[3], read_u64(ptr)); ptr += 8;
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 34) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 37) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 34) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 37) * k0;
}
if ((end - ptr) >= 16)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],34) * k3;
v[1] += read_u64(ptr) * k2; ptr += 8; v[1] = rotate_right(v[1],34) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 30) * k1;
v[1] ^= rotate_right((v[1] * k3) + v[0], 30) * k0;
}
if ((end - ptr) >= 8)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],36) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 23) * k1;
}
if ((end - ptr) >= 4)
{
v[1] ^= _mm_crc32_u64(v[0], read_u32(ptr)); ptr += 4;
v[1] ^= rotate_right((v[1] * k3) + v[0], 19) * k0;
}
if ((end - ptr) >= 2)
{
v[0] ^= _mm_crc32_u64(v[1], read_u16(ptr)); ptr += 2;
v[0] ^= rotate_right((v[0] * k2) + v[1], 13) * k1;
}
if ((end - ptr) >= 1)
{
v[1] ^= _mm_crc32_u64(v[0], read_u8 (ptr));
v[1] ^= rotate_right((v[1] * k3) + v[0], 17) * k0;
}
v[0] += rotate_right((v[0] * k0) + v[1], 11);
v[1] += rotate_right((v[1] * k1) + v[0], 26);
v[0] += rotate_right((v[0] * k0) + v[1], 11);
v[1] += rotate_right((v[1] * k1) + v[0], 26);
memcpy(out, v, 16);
}
void metrohash128crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xEE783E2F;
static const uint64_t k1 = 0xAD07C493;
static const uint64_t k2 = 0x797A90BB;
static const uint64_t k3 = 0x2E4B2E1B;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t v[4];
v[0] = ((static_cast<uint64_t>(seed) - k0) * k3) + len;
v[1] = ((static_cast<uint64_t>(seed) + k1) * k2) + len;
if (len >= 32)
{
v[2] = ((static_cast<uint64_t>(seed) + k0) * k2) + len;
v[3] = ((static_cast<uint64_t>(seed) - k1) * k3) + len;
do
{
v[0] ^= _mm_crc32_u64(v[0], read_u64(ptr)); ptr += 8;
v[1] ^= _mm_crc32_u64(v[1], read_u64(ptr)); ptr += 8;
v[2] ^= _mm_crc32_u64(v[2], read_u64(ptr)); ptr += 8;
v[3] ^= _mm_crc32_u64(v[3], read_u64(ptr)); ptr += 8;
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 12) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 19) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 12) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 19) * k0;
}
if ((end - ptr) >= 16)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],41) * k3;
v[1] += read_u64(ptr) * k2; ptr += 8; v[1] = rotate_right(v[1],41) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 10) * k1;
v[1] ^= rotate_right((v[1] * k3) + v[0], 10) * k0;
}
if ((end - ptr) >= 8)
{
v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],34) * k3;
v[0] ^= rotate_right((v[0] * k2) + v[1], 22) * k1;
}
if ((end - ptr) >= 4)
{
v[1] ^= _mm_crc32_u64(v[0], read_u32(ptr)); ptr += 4;
v[1] ^= rotate_right((v[1] * k3) + v[0], 14) * k0;
}
if ((end - ptr) >= 2)
{
v[0] ^= _mm_crc32_u64(v[1], read_u16(ptr)); ptr += 2;
v[0] ^= rotate_right((v[0] * k2) + v[1], 15) * k1;
}
if ((end - ptr) >= 1)
{
v[1] ^= _mm_crc32_u64(v[0], read_u8 (ptr));
v[1] ^= rotate_right((v[1] * k3) + v[0], 18) * k0;
}
v[0] += rotate_right((v[0] * k0) + v[1], 15);
v[1] += rotate_right((v[1] * k1) + v[0], 27);
v[0] += rotate_right((v[0] * k0) + v[1], 15);
v[1] += rotate_right((v[1] * k1) + v[0], 27);
memcpy(out, v, 16);
}

182
libs/libmetrohash/src/metrohash64.cpp Normal file
View File

@ -0,0 +1,182 @@
// metrohash64.cpp
//
// The MIT License (MIT)
//
// Copyright (c) 2015 J. Andrew Rogers
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
#include "metrohash.h"
void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xC83A91E1;
static const uint64_t k1 = 0x8648DBDB;
static const uint64_t k2 = 0x7BDEC03B;
static const uint64_t k3 = 0x2F5870A5;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t hash = ((static_cast<uint64_t>(seed) + k2) * k0) + len;
if (len >= 32)
{
uint64_t v[4];
v[0] = hash;
v[1] = hash;
v[2] = hash;
v[3] = hash;
do
{
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 33) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 33) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 33) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 33) * k0;
hash += v[0] ^ v[1];
}
if ((end - ptr) >= 16)
{
uint64_t v0 = hash + (read_u64(ptr) * k0); ptr += 8; v0 = rotate_right(v0,33) * k1;
uint64_t v1 = hash + (read_u64(ptr) * k1); ptr += 8; v1 = rotate_right(v1,33) * k2;
v0 ^= rotate_right(v0 * k0, 35) + v1;
v1 ^= rotate_right(v1 * k3, 35) + v0;
hash += v1;
}
if ((end - ptr) >= 8)
{
hash += read_u64(ptr) * k3; ptr += 8;
hash ^= rotate_right(hash, 33) * k1;
}
if ((end - ptr) >= 4)
{
hash += read_u32(ptr) * k3; ptr += 4;
hash ^= rotate_right(hash, 15) * k1;
}
if ((end - ptr) >= 2)
{
hash += read_u16(ptr) * k3; ptr += 2;
hash ^= rotate_right(hash, 13) * k1;
}
if ((end - ptr) >= 1)
{
hash += read_u8 (ptr) * k3;
hash ^= rotate_right(hash, 25) * k1;
}
hash ^= rotate_right(hash, 33);
hash *= k0;
hash ^= rotate_right(hash, 33);
memcpy(out, &hash, 8);
}
void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
{
static const uint64_t k0 = 0xD6D018F5;
static const uint64_t k1 = 0xA2AA033B;
static const uint64_t k2 = 0x62992FC1;
static const uint64_t k3 = 0x30BC5B29;
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
const uint8_t * const end = ptr + len;
uint64_t hash = ((static_cast<uint64_t>(seed) + k2) * k0) + len;
if (len >= 32)
{
uint64_t v[4];
v[0] = hash;
v[1] = hash;
v[2] = hash;
v[3] = hash;
do
{
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
}
while (ptr <= (end - 32));
v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 30) * k1;
v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 30) * k0;
v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 30) * k1;
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 30) * k0;
hash += v[0] ^ v[1];
}
if ((end - ptr) >= 16)
{
uint64_t v0 = hash + (read_u64(ptr) * k2); ptr += 8; v0 = rotate_right(v0,29) * k3;
uint64_t v1 = hash + (read_u64(ptr) * k2); ptr += 8; v1 = rotate_right(v1,29) * k3;
v0 ^= rotate_right(v0 * k0, 34) + v1;
v1 ^= rotate_right(v1 * k3, 34) + v0;
hash += v1;
}
if ((end - ptr) >= 8)
{
hash += read_u64(ptr) * k3; ptr += 8;
hash ^= rotate_right(hash, 36) * k1;
}
if ((end - ptr) >= 4)
{
hash += read_u32(ptr) * k3; ptr += 4;
hash ^= rotate_right(hash, 15) * k1;
}
if ((end - ptr) >= 2)
{
hash += read_u16(ptr) * k3; ptr += 2;
hash ^= rotate_right(hash, 15) * k1;
}
if ((end - ptr) >= 1)
{
hash += read_u8 (ptr) * k3;
hash ^= rotate_right(hash, 23) * k1;
}
hash ^= rotate_right(hash, 28);
hash *= k0;
hash ^= rotate_right(hash, 29);
memcpy(out, &hash, 8);
}

70
libs/libmetrohash/src/testvector.h Normal file
View File

@ -0,0 +1,70 @@
// testvector.h
//
// The MIT License (MIT)
//
// Copyright (c) 2015 J. Andrew Rogers
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
#ifndef METROHASH_TESTVECTOR_H
#define METROHASH_TESTVECTOR_H
#include "metrohash.h"
typedef void (*HashFunction) (const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * hash);
struct TestVectorData
{
HashFunction function;
uint32_t bits;
const char * key;
uint32_t seed;
uint8_t hash[64];
};
// The test vector string is selected such that it will properly exercise every
// internal branch of the hash function. Currently that requires a string with
// a length of (at least) 63 bytes.
static const char * test_key_63 = "012345678901234567890123456789012345678901234567890123456789012";
const TestVectorData TestVector [] =
{
// seed = 0
{ metrohash64_1, 64, test_key_63, 0, "658F044F5C730E40" },
{ metrohash64_2, 64, test_key_63, 0, "073CAAB960623211" },
{ metrohash128_1, 128, test_key_63, 0, "ED9997ED9D0A8B0FF3F266399477788F" },
{ metrohash128_2, 128, test_key_63, 0, "7BBA6FE119CF35D45507EDF3505359AB" },
{ metrohash128crc_1, 128, test_key_63, 0, "B329ED67831604D3DFAC4E4876D8262F" },
{ metrohash128crc_2, 128, test_key_63, 0, "0502A67E257BBD77206BBCA6BBEF2653" },
// seed = 1
{ metrohash64_1, 64, test_key_63, 1, "AE49EBB0A856537B" },
{ metrohash64_2, 64, test_key_63, 1, "CF518E9CF58402C0" },
{ metrohash128_1, 128, test_key_63, 1, "DDA6BA67F7DE755EFDF6BEABECCFD1F4" },
{ metrohash128_2, 128, test_key_63, 1, "2DA6AF149A5CDBC12B09DB0846D69EF0" },
{ metrohash128crc_1, 128, test_key_63, 1, "E8FAB51AF19F18A7B10D0A57D4276DF2" },
{ metrohash128crc_2, 128, test_key_63, 1, "2D54F87181A0CF64B02C50D95692BC19" },
};
#endif // #ifndef METROHASH_TESTVECTOR_H