thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Post-PR fixes stage 3

Browse Source 
maskLowBits()
building for clang
more test cases for codecs.
This commit is contained in:
Vasily Nemkov 2019-06-19 13:38:15 +03:00
parent 81f9055d81
commit e79c3d0e22
4 changed files with 149 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/src/Common/BitHelpers.h
View File

@ -79,5 +79,5 @@ inline size_t getTrailingZeroBits(T x)
template <typename T>
inline T maskLowBits(unsigned char bits)
{
return static_cast<T>(static_cast<T>(1) << bits) - 1;
return static_cast<T>(static_cast<T>(~T{0}) >> (sizeof(T) * 8 - bits));
}

192
dbms/src/Compression/tests/gtest_compressionCodec.cpp
View File

@ -3,13 +3,22 @@
#include <Core/Types.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromMemory.h>
#include <Common/PODArray.h>
#include <boost/format.hpp>
#include <cmath>
#include <initializer_list>
#include <iomanip>
#include <memory>
#include <vector>
#include <typeinfo>
#include <iterator>
#include <optional>
#include <iostream>
#include <bitset>
#include <string.h>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
@ -18,49 +27,16 @@
#endif
#include <gtest/gtest.h>
namespace
{
using namespace DB;
template <typename ContainerLeft, typename ContainerRight>
::testing::AssertionResult EqualContainers(const ContainerLeft & left, const ContainerRight & right)
template <typename T>
std::string bin(const T & value, size_t bits = sizeof(T)*8)
{
const auto MAX_MISMATCHING_ITEMS = 5;
static const UInt8 MAX_BITS = sizeof(T)*8;
assert(bits <= MAX_BITS);
const auto l_size = std::size(left);
const auto r_size = std::size(right);
const auto size = std::min(l_size, r_size);
::testing::AssertionResult result = ::testing::AssertionSuccess();
size_t mismatching_items = 0;
if (l_size != r_size)
{
result = ::testing::AssertionFailure() << "size mismatch" << " expected: " << l_size << " got:" << r_size;
}
for (size_t i = 0; i < size; ++i)
{
if (left[i] != right[i])
{
if (result)
{
result = ::testing::AssertionFailure();
}
result << "pos " << i << ": "
<< " expected: " << std::hex << left[i]
<< " got:" << std::hex << right[i]
<< std::endl;
if (++mismatching_items >= MAX_MISMATCHING_ITEMS)
{
result << "..." << std::endl;
break;
}
}
}
return result;
return std::bitset<sizeof(T) * 8>(static_cast<unsigned long long>(value))
.to_string().substr(MAX_BITS - bits, bits);
}
template <typename T>
@ -93,6 +69,72 @@ const char* type_name<Int64>()
return "int64";
}
template <>
const char* type_name<Float32>()
{
return "float";
}
template <>
const char* type_name<Float64>()
{
return "double";
}
template <typename T, typename ContainerLeft, typename ContainerRight>
::testing::AssertionResult EqualByteContainersAs(const ContainerLeft & left, const ContainerRight & right)
{
static_assert(sizeof(typename ContainerLeft::value_type) == 1, "Expected byte-container");
static_assert(sizeof(typename ContainerRight::value_type) == 1, "Expected byte-container");
::testing::AssertionResult result = ::testing::AssertionSuccess();
ReadBufferFromMemory left_read_buffer(left.data(), left.size());
ReadBufferFromMemory right_read_buffer(right.data(), right.size());
const auto l_size = left.size() / sizeof(T);
const auto r_size = right.size() / sizeof(T);
const auto size = std::min(l_size, r_size);
if (l_size != r_size)
{
result = ::testing::AssertionFailure() << "size mismatch" << " expected: " << l_size << " got:" << r_size;
}
const auto MAX_MISMATCHING_ITEMS = 5;
int mismatching_items = 0;
for (int i = 0; i < size; ++i)
{
T left_value{};
left_read_buffer.readStrict(reinterpret_cast<char*>(&left_value), sizeof(left_value));
T right_value{};
right_read_buffer.readStrict(reinterpret_cast<char*>(&right_value), sizeof(right_value));
if (left_value != right_value)
{
if (result)
{
result = ::testing::AssertionFailure();
}
result << "mismatching " << sizeof(T) << "-byte item #" << i
<< "\nexpected: " << bin(left_value)
<< "\ngot : " << bin(right_value)
<< std::endl;
if (++mismatching_items >= MAX_MISMATCHING_ITEMS)
{
result << "..." << std::endl;
break;
}
}
}
return result;
}
struct CodecTestParam
{
std::vector<char> source_data;
@ -107,15 +149,6 @@ std::ostream & operator<<(std::ostream & ostr, const CodecTestParam & param)
<< "\ndata size: " << param.source_data.size();
}
template <typename... Args>
std::string to_string(Args && ... args)
{
std::ostringstream ostr;
(ostr << ... << std::forward<Args>(args));
return ostr.str();
}
template <typename T, typename... Args>
CodecTestParam makeParam(Args && ... args)
{
@ -129,10 +162,11 @@ CodecTestParam makeParam(Args && ... args)
write_pos += sizeof(v);
}
return CodecTestParam{std::move(data), sizeof(T), to_string(data.size(), " predefined values")};
return CodecTestParam{std::move(data), sizeof(T),
(boost::format("%1% %2%") % data.size() % " predefined values").str()};
}
template <typename T, size_t Begin = 1, size_t End = 10, typename Generator>
template <typename T, size_t Begin = 1, size_t End = 10000, typename Generator>
CodecTestParam generateParam(Generator gen, const char* gen_name)
{
static_assert (End >= Begin, "End must be not less than Begin");
@ -148,7 +182,7 @@ CodecTestParam generateParam(Generator gen, const char* gen_name)
}
return CodecTestParam{std::move(data), sizeof(T),
to_string(type_name<T>(), " from ", gen_name, "(", Begin, " => ", End, ")")};
(boost::format("%1% from %2% (%3% => %4%)") % type_name<T>() % gen_name % Begin % End).str()};
}
void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param)
@ -165,11 +199,34 @@ void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param)
const UInt32 decoded_size = codec->decompress(encoded.data(), encoded.size(), decoded.data());
decoded.resize(decoded_size);
ASSERT_TRUE(EqualContainers(source_data, decoded));
switch (param.data_byte_size)
{
case 1:
ASSERT_TRUE(EqualByteContainersAs<UInt8>(source_data, decoded));
break;
case 2:
ASSERT_TRUE(EqualByteContainersAs<UInt16>(source_data, decoded));
break;
case 4:
ASSERT_TRUE(EqualByteContainersAs<UInt32>(source_data, decoded));
break;
case 8:
ASSERT_TRUE(EqualByteContainersAs<UInt64>(source_data, decoded));
break;
default:
FAIL() << "Invalid data_byte_size: " << param.data_byte_size;
}
}
class CodecTest : public ::testing::TestWithParam<CodecTestParam>
{};
{
public:
static void SetUpTestCase()
{
// To make random predicatble and avoid failing test "out of the blue".
srand(0);
}
};
TEST_P(CodecTest, DoubleDelta)
{
@ -246,6 +303,11 @@ auto MinMaxGenerator = [](auto i)
auto RandomGenerator = [](auto i) {return static_cast<decltype(i)>(rand());};
auto RandomishGenerator = [](auto i)
{
return static_cast<decltype(i)>(sin(static_cast<double>(i) * i) * i);
};
INSTANTIATE_TEST_CASE_P(Basic,
CodecTest,
::testing::Values(
@ -253,7 +315,7 @@ INSTANTIATE_TEST_CASE_P(Basic,
makeParam<UInt64>(1, 2, 3, 4),
makeParam<Float32>(1.1, 2.2, 3.3, 4.4),
makeParam<Float64>(1.1, 2.2, 3.3, 4.4)
)
),
);
#define G(generator) generator, #generator
@ -267,7 +329,7 @@ INSTANTIATE_TEST_CASE_P(Same,
generateParam<Int64>(G(SameValueGenerator(-1000))),
generateParam<Float32>(G(SameValueGenerator(M_E))),
generateParam<Float64>(G(SameValueGenerator(M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(Sequential,
@ -279,7 +341,7 @@ INSTANTIATE_TEST_CASE_P(Sequential,
generateParam<Int64>(G(SequentialGenerator(-1))),
generateParam<Float32>(G(SequentialGenerator(M_E))),
generateParam<Float64>(G(SequentialGenerator(M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(Monotonic,
@ -291,7 +353,7 @@ INSTANTIATE_TEST_CASE_P(Monotonic,
generateParam<Int64>(G(MonotonicGenerator<Int64>(-1, 5))),
generateParam<Float32>(G(MonotonicGenerator<Float32>(M_E, 5))),
generateParam<Float64>(G(MonotonicGenerator<Float64>(M_E, 5)))
)
),
);
INSTANTIATE_TEST_CASE_P(Random,
@ -299,7 +361,17 @@ INSTANTIATE_TEST_CASE_P(Random,
::testing::Values(
generateParam<UInt32>(G(RandomGenerator)),
generateParam<UInt64>(G(RandomGenerator))
)
),
);
INSTANTIATE_TEST_CASE_P(RandomLike,
CodecTest,
::testing::Values(
generateParam<Int32>(G(RandomishGenerator)),
generateParam<Int64>(G(RandomishGenerator)),
generateParam<Float32>(G(RandomishGenerator)),
generateParam<Float64>(G(RandomishGenerator))
),
);
INSTANTIATE_TEST_CASE_P(Overflow,
@ -309,7 +381,5 @@ INSTANTIATE_TEST_CASE_P(Overflow,
generateParam<Int32>(G(MinMaxGenerator)),
generateParam<UInt64>(G(MinMaxGenerator)),
generateParam<Int64>(G(MinMaxGenerator))
)
),
);
}

25
dbms/src/IO/tests/gtest_bit_io.cpp
View File

@ -1,3 +1,4 @@
#include <string.h>
#include <IO/BitHelpers.h>
#include <Core/Types.h>
@ -6,10 +7,14 @@
#include <Common/BitHelpers.h>
#include <Common/PODArray.h>
#include <memory>
#include <iostream>
#include <cmath>
#include <iomanip>
#include <memory>
#include <bitset>
#include <string>
#include <vector>
#include <typeinfo>
#include <iostream>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
@ -19,9 +24,6 @@
#include <gtest/gtest.h>
namespace
{
using namespace DB;
// Intentionally asymmetric both byte and word-size to detect read and write inconsistencies
@ -168,7 +170,7 @@ INSTANTIATE_TEST_CASE_P(Simple,
TestCaseParameter({{33, BIT_PATTERN}, {33, BIT_PATTERN}}),
TestCaseParameter({{24, 0xFFFFFFFF}},
"11111111 11111111 11111111 ")
));
),);
TestCaseParameter primes_case(UInt8 repeat_times, UInt64 pattern)
{
@ -192,11 +194,9 @@ INSTANTIATE_TEST_CASE_P(Primes,
::testing::Values(
primes_case(11, 0xFFFFFFFFFFFFFFFFULL),
primes_case(11, BIT_PATTERN)
));
),);
}
TEST(BitHelpers, MaskLowBits)
TEST(BitHelpers, maskLowBits)
{
EXPECT_EQ(0b00000111, ::maskLowBits<UInt8>(3));
EXPECT_EQ(0b01111111, ::maskLowBits<UInt8>(7));
@ -205,4 +205,9 @@ TEST(BitHelpers, MaskLowBits)
EXPECT_EQ(0b00000111111111111111111111111111, ::maskLowBits<UInt32>(27));
EXPECT_EQ(0b111111111111111111111111111111111, ::maskLowBits<UInt64>(33));
EXPECT_EQ(0b11111111111111111111111111111111111, ::maskLowBits<UInt64>(35));
EXPECT_EQ(0xFF, ::maskLowBits<UInt8>(8));
EXPECT_EQ(0xFFFF, ::maskLowBits<UInt16>(16));
EXPECT_EQ(0xFFFFFFFF, ::maskLowBits<UInt32>(32));
EXPECT_EQ(0xFFFFFFFFFFFFFFFF, ::maskLowBits<UInt64>(64));
}

4
dbms/tests/queries/0_stateless/00950_test_gorilla_codec.sql
View File

@ -22,9 +22,9 @@ INSERT INTO codecTest (key, ref_valueF64, valueF64, ref_valueF32, valueF32)
INSERT INTO codecTest (key, ref_valueF64, valueF64, ref_valueF32, valueF32)
SELECT number AS n, n*sqrt(n) AS v, v, v, v FROM system.numbers LIMIT 201, 100;
-- worst case - random values
-- worst case - almost like a random values
INSERT INTO codecTest (key, ref_valueF64, valueF64, ref_valueF32, valueF32)
SELECT number AS n, (rand64() - 9223372036854775808)/10000000000000 AS v, v, v, v FROM system.numbers LIMIT 3001, 100;
SELECT number AS n, sin(n * n * n ) * n AS v, v, v, v FROM system.numbers LIMIT 3001, 100;
-- These floating-point values are expected to be BINARY equal, hence comparing the values are safe.