mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-15 10:52:30 +00:00
7beb58b0cf
* impl for uniqExact
* rm unused (read|write)Text methods
* fix style
* small fixes
* impl for variadic uniqExact
* refactor
* fix style
* more agressive inlining
* disable if max_threads=1
* small improvements
* review fixes
* Revert "rm unused (read|write)Text methods"
This reverts commit a7e7480584.
* encapsulate is_able_to_parallelize_merge in Data
* encapsulate is_exact & argument_is_tuple in Data
430 lines
11 KiB
C++
430 lines
11 KiB
C++
#include <iomanip>
|
|
#include <iostream>
|
|
|
|
#include <Interpreters/AggregationCommon.h>
|
|
|
|
#include <Common/HashTable/HashMap.h>
|
|
#include <Common/HashTable/HashSet.h>
|
|
#include <Common/HashTable/Hash.h>
|
|
|
|
#include <IO/ReadBufferFromString.h>
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
|
|
using namespace DB;
|
|
|
|
namespace
|
|
{
|
|
std::vector<UInt64> getVectorWithNumbersUpToN(size_t n)
|
|
{
|
|
std::vector<UInt64> res(n);
|
|
std::iota(res.begin(), res.end(), 0);
|
|
return res;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
/// To test dump functionality without using other hashes that can change
|
|
template <typename T>
|
|
struct DummyHash
|
|
{
|
|
size_t operator()(T key) const { return T(key); }
|
|
};
|
|
|
|
template<typename HashTable>
|
|
std::set<std::string> convertToSet(const HashTable & table)
|
|
{
|
|
std::set<std::string> result;
|
|
|
|
for (auto v: table)
|
|
result.emplace(toString(v.getValue()));
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
TEST(HashTable, Insert)
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
|
|
ASSERT_EQ(cont.size(), 2);
|
|
}
|
|
|
|
TEST(HashTable, Emplace)
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
Cont::LookupResult it;
|
|
bool inserted = false;
|
|
cont.emplace(1, it, inserted);
|
|
ASSERT_EQ(it->getKey(), 1);
|
|
ASSERT_EQ(inserted, true);
|
|
|
|
cont.emplace(2, it, inserted);
|
|
ASSERT_EQ(it->getKey(), 2);
|
|
ASSERT_EQ(inserted, true);
|
|
|
|
cont.emplace(1, it, inserted);
|
|
ASSERT_EQ(it->getKey(), 1);
|
|
ASSERT_EQ(inserted, false);
|
|
}
|
|
|
|
TEST(HashTable, Lookup)
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
|
|
Cont::LookupResult it = cont.find(1);
|
|
ASSERT_TRUE(it != nullptr);
|
|
|
|
it = cont.find(2);
|
|
ASSERT_TRUE(it != nullptr);
|
|
|
|
it = cont.find(3);
|
|
ASSERT_TRUE(it == nullptr);
|
|
}
|
|
|
|
TEST(HashTable, Iteration)
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
cont.insert(3);
|
|
|
|
std::set<std::string> expected = {"1", "2", "3"};
|
|
std::set<std::string> actual = convertToSet(cont);
|
|
|
|
ASSERT_EQ(actual, expected);
|
|
}
|
|
|
|
TEST(HashTable, Erase)
|
|
{
|
|
{
|
|
/// Check zero element deletion
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
cont.insert(0);
|
|
|
|
ASSERT_TRUE(cont.find(0) != nullptr && cont.find(0)->getKey() == 0);
|
|
|
|
cont.erase(0);
|
|
|
|
ASSERT_TRUE(cont.find(0) == nullptr);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [.(1)..............] erase of (1).
|
|
cont.insert(1);
|
|
|
|
ASSERT_TRUE(cont.find(1) != nullptr && cont.find(1)->getKey() == 1);
|
|
|
|
cont.erase(1);
|
|
|
|
ASSERT_TRUE(cont.find(1) == nullptr);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [.(1)(2)(3)............] erase of (1) does not break search for (2) (3).
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
cont.insert(3);
|
|
cont.erase(1);
|
|
|
|
ASSERT_TRUE(cont.find(1) == nullptr);
|
|
ASSERT_TRUE(cont.find(2) != nullptr && cont.find(2)->getKey() == 2);
|
|
ASSERT_TRUE(cont.find(3) != nullptr && cont.find(3)->getKey() == 3);
|
|
|
|
cont.erase(2);
|
|
cont.erase(3);
|
|
ASSERT_TRUE(cont.find(2) == nullptr);
|
|
ASSERT_TRUE(cont.find(3) == nullptr);
|
|
ASSERT_EQ(cont.size(), 0);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [.(1)(17).............] erase of (1) breaks search for (17) because their natural position is 1.
|
|
cont.insert(1);
|
|
cont.insert(17);
|
|
cont.erase(1);
|
|
|
|
ASSERT_TRUE(cont.find(1) == nullptr);
|
|
ASSERT_TRUE(cont.find(17) != nullptr && cont.find(17)->getKey() == 17);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [.(1)(2)(3)(17)...........] erase of (2) breaks search for (17) because their natural position is 1.
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
cont.insert(3);
|
|
cont.insert(17);
|
|
cont.erase(2);
|
|
|
|
ASSERT_TRUE(cont.find(2) == nullptr);
|
|
ASSERT_TRUE(cont.find(1) != nullptr && cont.find(1)->getKey() == 1);
|
|
ASSERT_TRUE(cont.find(3) != nullptr && cont.find(3)->getKey() == 3);
|
|
ASSERT_TRUE(cont.find(17) != nullptr && cont.find(17)->getKey() == 17);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [(16)(30)............(14)(15)] erase of (16) breaks search for (30) because their natural position is 14.
|
|
cont.insert(14);
|
|
cont.insert(15);
|
|
cont.insert(16);
|
|
cont.insert(30);
|
|
cont.erase(16);
|
|
|
|
ASSERT_TRUE(cont.find(16) == nullptr);
|
|
ASSERT_TRUE(cont.find(14) != nullptr && cont.find(14)->getKey() == 14);
|
|
ASSERT_TRUE(cont.find(15) != nullptr && cont.find(15)->getKey() == 15);
|
|
ASSERT_TRUE(cont.find(30) != nullptr && cont.find(30)->getKey() == 30);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<4>>;
|
|
Cont cont;
|
|
|
|
/// [(16)(30)............(14)(15)] erase of (15) breaks search for (30) because their natural position is 14.
|
|
cont.insert(14);
|
|
cont.insert(15);
|
|
cont.insert(16);
|
|
cont.insert(30);
|
|
cont.erase(15);
|
|
|
|
ASSERT_TRUE(cont.find(15) == nullptr);
|
|
ASSERT_TRUE(cont.find(14) != nullptr && cont.find(14)->getKey() == 14);
|
|
ASSERT_TRUE(cont.find(16) != nullptr && cont.find(16)->getKey() == 16);
|
|
ASSERT_TRUE(cont.find(30) != nullptr && cont.find(30)->getKey() == 30);
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
Cont cont;
|
|
|
|
for (int i = 0; i < 5000; ++i)
|
|
{
|
|
cont.insert(i);
|
|
}
|
|
|
|
for (int i = 0; i < 2500; ++i)
|
|
{
|
|
cont.erase(i);
|
|
}
|
|
|
|
for (int i = 5000; i < 10000; ++i)
|
|
{
|
|
cont.insert(i);
|
|
}
|
|
|
|
for (int i = 5000; i < 10000; ++i)
|
|
{
|
|
cont.erase(i);
|
|
}
|
|
|
|
for (int i = 2500; i < 5000; ++i)
|
|
{
|
|
cont.erase(i);
|
|
}
|
|
|
|
ASSERT_EQ(cont.size(), 0);
|
|
}
|
|
}
|
|
|
|
TEST(HashTable, SerializationDeserialization)
|
|
{
|
|
{
|
|
/// Use dummy hash to make it reproducible if default hash implementation will be changed
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
cont.insert(3);
|
|
|
|
WriteBufferFromOwnString wb;
|
|
cont.writeText(wb);
|
|
|
|
std::string expected = "3,1,2,3";
|
|
|
|
ASSERT_EQ(wb.str(), expected);
|
|
|
|
ReadBufferFromString rb(expected);
|
|
|
|
Cont deserialized;
|
|
deserialized.readText(rb);
|
|
ASSERT_EQ(convertToSet(cont), convertToSet(deserialized));
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DefaultHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
|
|
Cont cont;
|
|
|
|
cont.insert(1);
|
|
cont.insert(2);
|
|
cont.insert(3);
|
|
|
|
WriteBufferFromOwnString wb;
|
|
cont.write(wb);
|
|
|
|
ReadBufferFromString rb(wb.str());
|
|
|
|
Cont deserialized;
|
|
deserialized.read(rb);
|
|
ASSERT_EQ(convertToSet(cont), convertToSet(deserialized));
|
|
}
|
|
{
|
|
using Cont = HashSet<int, DummyHash<int>, HashTableGrowerWithPrecalculation<1>>;
|
|
Cont cont;
|
|
|
|
WriteBufferFromOwnString wb;
|
|
cont.writeText(wb);
|
|
|
|
std::string expected = "0";
|
|
ASSERT_EQ(wb.str(), expected);
|
|
|
|
ReadBufferFromString rb(expected);
|
|
|
|
Cont deserialized;
|
|
deserialized.readText(rb);
|
|
ASSERT_EQ(convertToSet(cont), convertToSet(deserialized));
|
|
}
|
|
{
|
|
using Cont = HashSet<UInt128, UInt128TrivialHash>;
|
|
Cont cont;
|
|
|
|
WriteBufferFromOwnString wb;
|
|
cont.write(wb);
|
|
|
|
std::string expected;
|
|
expected += static_cast<char>(0);
|
|
|
|
ASSERT_EQ(wb.str(), expected);
|
|
|
|
ReadBufferFromString rb(expected);
|
|
|
|
Cont deserialized;
|
|
deserialized.read(rb);
|
|
ASSERT_EQ(convertToSet(cont), convertToSet(deserialized));
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
struct IdentityHash
|
|
{
|
|
size_t operator()(T x) const { return x; }
|
|
};
|
|
|
|
struct OneElementResizeGrower
|
|
{
|
|
/// If collision resolution chains are contiguous, we can implement erase operation by moving the elements.
|
|
static constexpr auto performs_linear_probing_with_single_step = true;
|
|
|
|
static constexpr size_t initial_count = 1;
|
|
|
|
size_t bufSize() const { return buf_size; }
|
|
|
|
size_t place(size_t x) const { return x % buf_size; }
|
|
|
|
size_t next(size_t pos) const { return (pos + 1) % buf_size; }
|
|
|
|
bool overflow(size_t elems) const { return elems >= buf_size; }
|
|
|
|
void increaseSize() { ++buf_size; }
|
|
|
|
void set(size_t) { }
|
|
|
|
void setBufSize(size_t buf_size_) { buf_size = buf_size_; }
|
|
|
|
size_t buf_size = initial_count;
|
|
};
|
|
|
|
TEST(HashTable, Resize)
|
|
{
|
|
{
|
|
/// Test edge case if after resize all cells are resized in end of buf and will take half of
|
|
/// hash table place.
|
|
using HashSet = HashSet<int, IdentityHash<int>, OneElementResizeGrower>;
|
|
HashSet cont;
|
|
|
|
cont.insert(3);
|
|
cont.insert(1);
|
|
|
|
std::set<std::string> expected = {"1", "3"};
|
|
std::set<std::string> actual = convertToSet(cont);
|
|
|
|
ASSERT_EQ(actual, expected);
|
|
}
|
|
}
|
|
|
|
|
|
using HashSetContent = std::vector<UInt64>;
|
|
|
|
class TwoLevelHashSetFixture : public ::testing::TestWithParam<HashSetContent>
|
|
{
|
|
};
|
|
|
|
|
|
TEST_P(TwoLevelHashSetFixture, WriteAsSingleLevel)
|
|
{
|
|
using Key = UInt64;
|
|
|
|
{
|
|
const auto & hash_set_content = GetParam();
|
|
|
|
TwoLevelHashSet<Key, HashCRC32<Key>> two_level;
|
|
for (const auto & elem : hash_set_content)
|
|
two_level.insert(elem);
|
|
|
|
WriteBufferFromOwnString wb;
|
|
two_level.writeAsSingleLevel(wb);
|
|
|
|
ReadBufferFromString rb(wb.str());
|
|
HashSet<Key, HashCRC32<Key>> single_level;
|
|
single_level.read(rb);
|
|
|
|
EXPECT_EQ(single_level.size(), hash_set_content.size());
|
|
for (const auto & elem : hash_set_content)
|
|
EXPECT_NE(single_level.find(elem), nullptr);
|
|
}
|
|
}
|
|
|
|
|
|
INSTANTIATE_TEST_SUITE_P(
|
|
TwoLevelHashSetTests,
|
|
TwoLevelHashSetFixture,
|
|
::testing::Values(
|
|
HashSetContent{},
|
|
getVectorWithNumbersUpToN(1),
|
|
getVectorWithNumbersUpToN(100),
|
|
getVectorWithNumbersUpToN(1000),
|
|
getVectorWithNumbersUpToN(10000),
|
|
getVectorWithNumbersUpToN(100000),
|
|
getVectorWithNumbersUpToN(1000000)));
|