thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-24 02:30:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Gorilla encoding error on small sequences.

Browse Source 
Added test cases for small sequences;
Refurbished test cases for codecs;
This commit is contained in:
Vasily Nemkov 2019-08-11 11:01:02 +03:00
parent dd6f48342d
commit 923c6889e8
3 changed files with 592 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
dbms/src/Compression/CompressionCodecGorilla.cpp
View File

@ -78,11 +78,12 @@ binary_value_info getLeadingAndTrailingBits(const T & value)
const UInt8 lz = getLeadingZeroBits(value);
const UInt8 tz = getTrailingZeroBits(value);
const UInt8 data_size = value == 0 ? 0 : static_cast<UInt8>(bit_size - lz - tz);
return binary_value_info{lz, data_size, tz};
}
template <typename T>
UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, UInt32 dest_size)
{
static const auto DATA_BIT_LENGTH = getBitLengthOfLength(sizeof(T));
// -1 since there must be at least 1 non-zero bit.
@ -91,6 +92,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
if (source_size % sizeof(T) != 0)
throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS);
const char * source_end = source + source_size;
const char * dest_end = dest + dest_size;
const UInt32 items_count = source_size / sizeof(T);
@ -110,7 +112,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
dest += sizeof(prev_value);
}
WriteBuffer buffer(dest, getCompressedDataSize(sizeof(T), source_size - sizeof(items_count) - sizeof(prev_value)));
WriteBuffer buffer(dest, dest_end - dest);
BitWriter writer(buffer);
while (source < source_end)
@ -265,24 +267,26 @@ UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 sourc
dest[1] = bytes_to_skip;
memcpy(&dest[2], source, bytes_to_skip);
size_t start_pos = 2 + bytes_to_skip;
UInt32 compressed_size = 0;
UInt32 result_size = 0;
const UInt32 compressed_size = getMaxCompressedDataSize(source_size);
switch (data_bytes_size)
{
case 1:
compressed_size = compressDataForType<UInt8>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]);
result_size = compressDataForType<UInt8>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size);
break;
case 2:
compressed_size = compressDataForType<UInt16>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]);
result_size = compressDataForType<UInt16>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size);
break;
case 4:
compressed_size = compressDataForType<UInt32>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]);
result_size = compressDataForType<UInt32>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size);
break;
case 8:
compressed_size = compressDataForType<UInt64>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]);
result_size = compressDataForType<UInt64>(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size);
break;
}
return 1 + 1 + compressed_size;
return 1 + 1 + result_size;
}
void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 /* uncompressed_size */) const

2
dbms/src/Compression/ICompressionCodec.cpp
View File

@ -49,8 +49,8 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch
UInt8 header_size = getHeaderSize();
UInt32 decompressed_size = unalignedLoad<UInt32>(&source[5]);
doDecompressData(&source[header_size], source_size - header_size, dest, decompressed_size);
return decompressed_size;
return decompressed_size;
}
UInt32 ICompressionCodec::readCompressedBlockSize(const char * source)

730
dbms/src/Compression/tests/gtest_compressionCodec.cpp
View File

@ -1,10 +1,13 @@
#include <Compression/CompressionCodecDoubleDelta.h>
#include <Compression/CompressionCodecGorilla.h>
#include <Compression/CompressionFactory.h>
#include <Core/Types.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromMemory.h>
#include <Common/PODArray.h>
#include <DataTypes/IDataType.h>
#include <DataTypes/DataTypesNumber.h>
#include <Parsers/ExpressionElementParsers.h>
#include <boost/format.hpp>
@ -24,6 +27,33 @@
using namespace DB;
namespace std
{
template <typename T>
std::ostream & operator<<(std::ostream & ostr, const std::optional<T> & opt)
{
if (!opt)
{
return ostr << "<empty optional>";
}
return ostr << *opt;
}
template <typename T>
std::vector<T> operator+(std::vector<T> && left, std::vector<T> && right)
{
std::vector<T> result(std::move(left));
std::move(std::begin(right), std::end(right), std::back_inserter(result));
return result;
}
}
namespace
{
template <typename T>
std::string bin(const T & value, size_t bits = sizeof(T)*8)
{
@ -37,43 +67,46 @@ std::string bin(const T & value, size_t bits = sizeof(T)*8)
template <typename T>
const char* type_name()
{
#define MAKE_TYPE_NAME(TYPE) \
if constexpr (std::is_same_v<TYPE, T>) return #TYPE;
MAKE_TYPE_NAME(UInt8);
MAKE_TYPE_NAME(UInt16);
MAKE_TYPE_NAME(UInt32);
MAKE_TYPE_NAME(UInt64);
MAKE_TYPE_NAME(Int8);
MAKE_TYPE_NAME(Int16);
MAKE_TYPE_NAME(Int32);
MAKE_TYPE_NAME(Int64);
MAKE_TYPE_NAME(Float32);
MAKE_TYPE_NAME(Float64);
#undef MAKE_TYPE_NAME
return typeid(T).name();
}
template <>
const char* type_name<UInt32>()
template <typename T>
DataTypePtr makeDataType()
{
return "uint32";
}
#define MAKE_DATA_TYPE(TYPE) \
if constexpr (std::is_same_v<T, TYPE>) return std::make_shared<DataType ## TYPE>()
template <>
const char* type_name<Int32>()
{
return "int32";
}
MAKE_DATA_TYPE(UInt8);
MAKE_DATA_TYPE(UInt16);
MAKE_DATA_TYPE(UInt32);
MAKE_DATA_TYPE(UInt64);
MAKE_DATA_TYPE(Int8);
MAKE_DATA_TYPE(Int16);
MAKE_DATA_TYPE(Int32);
MAKE_DATA_TYPE(Int64);
MAKE_DATA_TYPE(Float32);
MAKE_DATA_TYPE(Float64);
template <>
const char* type_name<UInt64>()
{
return "uint64";
}
#undef MAKE_DATA_TYPE
template <>
const char* type_name<Int64>()
{
return "int64";
}
template <>
const char* type_name<Float32>()
{
return "float";
}
template <>
const char* type_name<Float64>()
{
return "double";
assert(false && "unsupported size");
return nullptr;
}
@ -135,52 +168,83 @@ template <typename T, typename ContainerLeft, typename ContainerRight>
return result;
}
struct CodecTestParam
struct CodecTestSequence
{
std::string type_name;
std::vector<char> source_data;
UInt8 data_byte_size;
double min_compression_ratio;
std::string case_name;
std::string name;
std::vector<char> serialized_data;
// to allow setting ratio after building with complex builder functions.
CodecTestParam && setRatio(const double & ratio) &&
{
this->min_compression_ratio = ratio;
return std::move(*this);
}
DataTypePtr data_type;
};
CodecTestParam operator+(CodecTestParam && left, CodecTestParam && right)
struct Codec
{
assert(left.type_name == right.type_name);
assert(left.data_byte_size == right.data_byte_size);
std::string codec_statement;
std::optional<double> expected_compression_ratio;
std::vector data(std::move(left.source_data));
data.insert(data.end(), right.source_data.begin(), right.source_data.end());
explicit Codec(std::string codec_statement_, std::optional<double> expected_compression_ratio_ = std::nullopt)
: codec_statement(std::move(codec_statement_)),
expected_compression_ratio(expected_compression_ratio_)
{}
return CodecTestParam{
left.type_name,
Codec()
: Codec(std::string())
{}
};
CodecTestSequence operator+(CodecTestSequence && left, CodecTestSequence && right)
{
assert(left.data_type->equals(*right.data_type));
std::vector<char> data(std::move(left.serialized_data));
data.insert(data.end(), right.serialized_data.begin(), right.serialized_data.end());
return CodecTestSequence{
left.name + " + " + right.name,
std::move(data),
left.data_byte_size,
std::min(left.min_compression_ratio, right.min_compression_ratio),
left.case_name + " + " + right.case_name
std::move(left.data_type)
};
}
std::ostream & operator<<(std::ostream & ostr, const CodecTestParam & param)
template <typename T>
CodecTestSequence operator*(CodecTestSequence && left, T times)
{
return ostr << "name: " << param.case_name
<< "\ntype name:" << param.type_name
<< "\nbyte size: " << static_cast<UInt32>(param.data_byte_size)
<< "\ndata size: " << param.source_data.size();
std::vector<char> data(std::move(left.serialized_data));
const size_t initial_size = data.size();
const size_t final_size = initial_size * times;
data.reserve(final_size);
for (T i = 0; i < times; ++i)
{
data.insert(data.end(), data.begin(), data.begin() + initial_size);
}
// compression ratio < 1.0 means that codec output is smaller than input.
const double DEFAULT_MIN_COMPRESSION_RATIO = 1.0;
return CodecTestSequence{
left.name + " x " + std::to_string(times),
std::move(data),
std::move(left.data_type)
};
}
std::ostream & operator<<(std::ostream & ostr, const Codec & codec)
{
return ostr << "Codec{"
<< "name: " << codec.codec_statement
<< ", expected_compression_ratio: " << codec.expected_compression_ratio
<< "}";
}
std::ostream & operator<<(std::ostream & ostr, const CodecTestSequence & seq)
{
return ostr << "CodecTestSequence{"
<< "name: " << seq.name
<< ", type name: " << seq.data_type->getName()
<< ", data size: " << seq.serialized_data.size() << " bytes"
<< "}";
}
template <typename T, typename... Args>
CodecTestParam makeParam(Args && ... args)
CodecTestSequence makeSeq(Args && ... args)
{
std::initializer_list<T> vals{static_cast<T>(args)...};
std::vector<char> data(sizeof(T) * std::size(vals));
@ -192,14 +256,17 @@ CodecTestParam makeParam(Args && ... args)
write_pos += sizeof(v);
}
return CodecTestParam{type_name<T>(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO,
(boost::format("%1% values of %2%") % std::size(vals) % type_name<T>()).str()};
return CodecTestSequence{
(boost::format("%1% values of %2%") % std::size(vals) % type_name<T>()).str(),
std::move(data),
makeDataType<T>()
};
}
template <typename T, size_t Begin = 1, size_t End = 10001, typename Generator>
CodecTestParam generateParam(Generator gen, const char* gen_name)
template <typename T, typename Generator>
CodecTestSequence generateSeq(Generator gen, const char* gen_name, size_t Begin = 0, size_t End = 10000)
{
static_assert (End >= Begin, "End must be not less than Begin");
assert (End >= Begin);
std::vector<char> data(sizeof(T) * (End - Begin));
char * write_pos = data.data();
@ -211,25 +278,57 @@ CodecTestParam generateParam(Generator gen, const char* gen_name)
write_pos += sizeof(v);
}
return CodecTestParam{type_name<T>(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO,
(boost::format("%1% values of %2% from %3%") % (End - Begin) % type_name<T>() % gen_name).str()};
return CodecTestSequence{
(boost::format("%1% values of %2% from %3%") % (End - Begin) % type_name<T>() % gen_name).str(),
std::move(data),
makeDataType<T>()
};
}
void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param)
{
const auto & source_data = param.source_data;
const UInt32 encoded_max_size = codec->getCompressedReserveSize(source_data.size());
class CodecTest : public ::testing::TestWithParam<std::tuple<Codec, CodecTestSequence>>
{
public:
enum MakeCodecParam
{
CODEC_WITH_DATA_TYPE,
CODEC_WITHOUT_DATA_TYPE,
};
CompressionCodecPtr makeCodec(MakeCodecParam with_data_type) const
{
const auto & codec_string = std::get<0>(GetParam()).codec_statement;
const auto & data_type = with_data_type == CODEC_WITH_DATA_TYPE ? std::get<1>(GetParam()).data_type : nullptr;
const std::string codec_statement = "(" + codec_string + ")";
Tokens tokens(codec_statement.begin().base(), codec_statement.end().base());
TokenIterator token_iterator(tokens);
Expected expected;
ASTPtr codec_ast;
ParserCodec parser;
parser.parse(token_iterator, codec_ast, expected);
return CompressionCodecFactory::instance().get(codec_ast, data_type);
}
void testTranscoding(ICompressionCodec & codec)
{
const auto & test_sequence = std::get<1>(GetParam());
const auto & source_data = test_sequence.serialized_data;
const UInt32 encoded_max_size = codec.getCompressedReserveSize(source_data.size());
PODArray<char> encoded(encoded_max_size);
const UInt32 encoded_size = codec->compress(source_data.data(), source_data.size(), encoded.data());
const UInt32 encoded_size = codec.compress(source_data.data(), source_data.size(), encoded.data());
encoded.resize(encoded_size);
PODArray<char> decoded(source_data.size());
const UInt32 decoded_size = codec->decompress(encoded.data(), encoded.size(), decoded.data());
const UInt32 decoded_size = codec.decompress(encoded.data(), encoded.size(), decoded.data());
decoded.resize(decoded_size);
switch (param.data_byte_size)
switch (test_sequence.data_type->getSizeOfValueInMemory())
{
case 1:
ASSERT_TRUE(EqualByteContainersAs<UInt8>(source_data, decoded));
@ -244,56 +343,39 @@ void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param)
ASSERT_TRUE(EqualByteContainersAs<UInt64>(source_data, decoded));
break;
default:
FAIL() << "Invalid data_byte_size: " << param.data_byte_size;
FAIL() << "Invalid test sequence data type: " << test_sequence.data_type->getName();
}
const auto header_size = codec->getHeaderSize();
const auto header_size = codec.getHeaderSize();
const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0);
ASSERT_LE(compression_ratio, param.min_compression_ratio)
const auto & codec_spec = std::get<0>(GetParam());
if (codec_spec.expected_compression_ratio)
{
ASSERT_LE(compression_ratio, *codec_spec.expected_compression_ratio)
<< "\n\tdecoded size: " << source_data.size()
<< "\n\tencoded size: " << encoded_size
<< "(no header: " << encoded_size - header_size << ")";
}
class CodecTest : public ::testing::TestWithParam<CodecTestParam>
{
public:
static void SetUpTestCase()
{
// To make random predicatble and avoid failing test "out of the blue".
srand(0);
}
};
TEST_P(CodecTest, DoubleDelta)
TEST_P(CodecTest, TranscodingWithDataType)
{
auto param = GetParam();
auto codec = std::make_unique<CompressionCodecDoubleDelta>(param.data_byte_size);
if (param.type_name == type_name<Float32>() || param.type_name == type_name<Float64>())
{
// dd doesn't work great with many cases of integers and may result in very poor compression rate.
param.min_compression_ratio *= 1.5;
const auto codec = makeCodec(CODEC_WITH_DATA_TYPE);
testTranscoding(*codec);
}
TestTranscoding(codec.get(), param);
}
TEST_P(CodecTest, Gorilla)
TEST_P(CodecTest, TranscodingWithoutDataType)
{
auto param = GetParam();
auto codec = std::make_unique<CompressionCodecGorilla>(param.data_byte_size);
if (param.type_name == type_name<UInt32>() || param.type_name == type_name<Int32>()
|| param.type_name == type_name<UInt64>() || param.type_name == type_name<Int64>())
{
// gorilla doesn't work great with many cases of integers and may result in very poor compression rate.
param.min_compression_ratio *= 1.5;
}
TestTranscoding(codec.get(), param);
const auto codec = makeCodec(CODEC_WITHOUT_DATA_TYPE);
testTranscoding(*codec);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// Here we use generators to produce test payload for codecs.
// Generator is a callable that should produce output value of the same type as input value.
// Generator is a callable that can produce infinite number of values,
// output value MUST be of the same type input value.
///////////////////////////////////////////////////////////////////////////////////////////////////
auto SameValueGenerator = [](auto value)
{
@ -332,31 +414,65 @@ auto SequentialGenerator = [](auto stride = 1)
//};
template <typename T>
using uniform_distribution =
typename std::conditional_t<std::is_floating_point_v<T>, std::uniform_real_distribution<T>,
typename std::conditional_t<std::is_integral_v<T>, std::uniform_int_distribution<T>, void>>;
template <typename T = Int32>
struct MonotonicGenerator
{
MonotonicGenerator(T stride_ = 1, size_t max_step_ = 10)
MonotonicGenerator(T stride_ = 1, T max_step = 10)
: prev_value(0),
stride(stride_),
max_step(max_step_)
random_engine(0),
distribution(0, max_step)
{}
template <typename U>
U operator()(U)
{
const U result = prev_value + static_cast<T>(stride * (rand() % max_step));
prev_value = result;
return result;
prev_value = prev_value + stride * distribution(random_engine);
return static_cast<U>(prev_value);
}
private:
T prev_value;
const T stride;
const size_t max_step;
std::default_random_engine random_engine;
uniform_distribution<T> distribution;
};
auto MinMaxGenerator = [](auto i)
template <typename T>
struct RandomGenerator
{
if (i % 2 == 0)
RandomGenerator(T seed = 0, T value_min = std::numeric_limits<T>::min(), T value_max = std::numeric_limits<T>::max())
: random_engine(seed),
distribution(value_min, value_max)
{
}
template <typename U>
U operator()(U)
{
return static_cast<U>(distribution(random_engine));
}
private:
std::default_random_engine random_engine;
uniform_distribution<T> distribution;
};
auto RandomishGenerator = [](auto i)
{
return static_cast<decltype(i)>(sin(static_cast<double>(i * i)) * i);
};
auto MinMaxGenerator = []()
{
return [step = 0](auto i) mutable
{
if (step++ % 2 == 0)
{
return std::numeric_limits<decltype(i)>::min();
}
@ -365,108 +481,360 @@ auto MinMaxGenerator = [](auto i)
return std::numeric_limits<decltype(i)>::max();
}
};
template <typename T>
struct RandomGenerator
{
RandomGenerator(T seed = 0, T value_cap_ = std::numeric_limits<T>::max())
: e(seed),
value_cap(value_cap_)
{
}
template <typename U>
U operator()(U i)
{
return static_cast<decltype(i)>(distribution(e) % value_cap);
}
private:
std::default_random_engine e;
std::uniform_int_distribution<T> distribution;
const T value_cap;
};
auto RandomishGenerator = [](auto i)
// Fill dest value with 0x00 or 0xFF
auto FFand0Generator = []()
{
return static_cast<decltype(i)>(sin(static_cast<double>(i) * i) * i);
return [step = 0](auto i) mutable
{
decltype(i) result;
if (step++ % 2 == 0)
{
memset(&result, 0, sizeof(result));
}
else
{
memset(&result, 0xFF, sizeof(result));
}
return result;
};
};
// helper macro to produce human-friendly test case name
// Makes many sequences with generator, first sequence length is 1, second is 2... up to `sequences_count`.
template <typename T, typename Generator>
std::vector<CodecTestSequence> generatePyramidOfSequences(const size_t sequences_count, Generator && generator, const char* generator_name)
{
std::vector<CodecTestSequence> sequences;
sequences.reserve(sequences_count);
for (size_t i = 1; i < sequences_count; ++i)
{
std::string name = generator_name + std::string(" from 0 to ") + std::to_string(i);
sequences.push_back(generateSeq<T>(std::forward<decltype(generator)>(generator), name.c_str(), 0, i));
}
return sequences;
};
// helper macro to produce human-friendly sequence name from generator
#define G(generator) generator, #generator
const auto DefaultCodecsToTest = ::testing::Values(
Codec("DoubleDelta"),
Codec("DoubleDelta, LZ4"),
Codec("DoubleDelta, ZSTD"),
Codec("Gorilla"),
Codec("Gorilla, LZ4"),
Codec("Gorilla, ZSTD")
);
///////////////////////////////////////////////////////////////////////////////////////////////////
// test cases
///////////////////////////////////////////////////////////////////////////////////////////////////
INSTANTIATE_TEST_CASE_P(Simple,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
makeSeq<Float64>(1, 2, 3, 5, 7, 11, 13, 17, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97)
)
),
);
INSTANTIATE_TEST_CASE_P(SmallSequences,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::ValuesIn(
generatePyramidOfSequences<Int8 >(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<Int16 >(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<Int32 >(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<Int64 >(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<UInt8 >(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<UInt16>(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<UInt32>(42, G(SequentialGenerator(1)))
+ generatePyramidOfSequences<UInt64>(42, G(SequentialGenerator(1)))
)
),
);
INSTANTIATE_TEST_CASE_P(Mixed,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateParam<Int32, 1, 3>(G(MinMaxGenerator)) + generateParam<Int32, 1, 11>(G(SequentialGenerator(1))).setRatio(1),
generateParam<UInt32, 1, 3>(G(MinMaxGenerator)) + generateParam<UInt32, 1, 11>(G(SequentialGenerator(1))).setRatio(1),
generateParam<Int64, 1, 3>(G(MinMaxGenerator)) + generateParam<Int64, 1, 11>(G(SequentialGenerator(1))).setRatio(1),
generateParam<UInt64, 1, 3>(G(MinMaxGenerator)) + generateParam<UInt64, 1, 11>(G(SequentialGenerator(1))).setRatio(1)
generateSeq<Int8>(G(MinMaxGenerator()), 1, 5) + generateSeq<Int8>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<Int16>(G(MinMaxGenerator()), 1, 5) + generateSeq<Int16>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<Int32>(G(MinMaxGenerator()), 1, 5) + generateSeq<Int32>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<Int64>(G(MinMaxGenerator()), 1, 5) + generateSeq<Int64>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<UInt8>(G(MinMaxGenerator()), 1, 5) + generateSeq<UInt8>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<UInt16>(G(MinMaxGenerator()), 1, 5) + generateSeq<UInt16>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<UInt32>(G(MinMaxGenerator()), 1, 5) + generateSeq<UInt32>(G(SequentialGenerator(1)), 1, 1001),
generateSeq<UInt64>(G(MinMaxGenerator()), 1, 5) + generateSeq<UInt64>(G(SequentialGenerator(1)), 1, 1001)
)
),
);
INSTANTIATE_TEST_CASE_P(Same,
INSTANTIATE_TEST_CASE_P(SameValueInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateParam<UInt32>(G(SameValueGenerator(1000))),
generateParam<Int32>(G(SameValueGenerator(-1000))),
generateParam<UInt64>(G(SameValueGenerator(1000))),
generateParam<Int64>(G(SameValueGenerator(-1000))),
generateParam<Float32>(G(SameValueGenerator(M_E))),
generateParam<Float64>(G(SameValueGenerator(M_E)))
generateSeq<Int8 >(G(SameValueGenerator(1000))),
generateSeq<Int16 >(G(SameValueGenerator(1000))),
generateSeq<Int32 >(G(SameValueGenerator(1000))),
generateSeq<Int64 >(G(SameValueGenerator(1000))),
generateSeq<UInt8 >(G(SameValueGenerator(1000))),
generateSeq<UInt16>(G(SameValueGenerator(1000))),
generateSeq<UInt32>(G(SameValueGenerator(1000))),
generateSeq<UInt64>(G(SameValueGenerator(1000)))
)
),
);
INSTANTIATE_TEST_CASE_P(Sequential,
INSTANTIATE_TEST_CASE_P(SameNegativeValueInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateParam<UInt32>(G(SequentialGenerator(1))),
generateParam<Int32>(G(SequentialGenerator(-1))),
generateParam<UInt64>(G(SequentialGenerator(1))),
generateParam<Int64>(G(SequentialGenerator(-1))),
generateParam<Float32>(G(SequentialGenerator(M_E))),
generateParam<Float64>(G(SequentialGenerator(M_E)))
generateSeq<Int8 >(G(SameValueGenerator(-1000))),
generateSeq<Int16 >(G(SameValueGenerator(-1000))),
generateSeq<Int32 >(G(SameValueGenerator(-1000))),
generateSeq<Int64 >(G(SameValueGenerator(-1000))),
generateSeq<UInt8 >(G(SameValueGenerator(-1000))),
generateSeq<UInt16>(G(SameValueGenerator(-1000))),
generateSeq<UInt32>(G(SameValueGenerator(-1000))),
generateSeq<UInt64>(G(SameValueGenerator(-1000)))
)
),
);
INSTANTIATE_TEST_CASE_P(Monotonic,
INSTANTIATE_TEST_CASE_P(SameValueFloat,
CodecTest,
::testing::Combine(
::testing::Values(
generateParam<UInt32>(G(MonotonicGenerator<UInt32>(1, 5))),
generateParam<Int32>(G(MonotonicGenerator<Int32>(-1, 5))),
generateParam<UInt64>(G(MonotonicGenerator<UInt64>(1, 5))),
generateParam<Int64>(G(MonotonicGenerator<Int64>(-1, 5))),
generateParam<Float32>(G(MonotonicGenerator<Float32>(M_E, 5))),
generateParam<Float64>(G(MonotonicGenerator<Float64>(M_E, 5)))
Codec("Gorilla"),
Codec("Gorilla, LZ4")
),
::testing::Values(
generateSeq<Float32>(G(SameValueGenerator(M_E))),
generateSeq<Float64>(G(SameValueGenerator(M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(Random,
INSTANTIATE_TEST_CASE_P(SameNegativeValueFloat,
CodecTest,
::testing::Combine(
::testing::Values(
generateParam<UInt32>(G(RandomGenerator<UInt32>(0, 1000'000'000))).setRatio(1.2),
generateParam<UInt64>(G(RandomGenerator<UInt64>(0, 1000'000'000))).setRatio(1.1)
Codec("Gorilla"),
Codec("Gorilla, LZ4")
),
::testing::Values(
generateSeq<Float32>(G(SameValueGenerator(-1 * M_E))),
generateSeq<Float64>(G(SameValueGenerator(-1 * M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(Randomish,
INSTANTIATE_TEST_CASE_P(SequentialInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateParam<Int32>(G(RandomishGenerator)).setRatio(1.1),
generateParam<Int64>(G(RandomishGenerator)).setRatio(1.1),
generateParam<UInt32>(G(RandomishGenerator)).setRatio(1.1),
generateParam<UInt64>(G(RandomishGenerator)).setRatio(1.1),
generateParam<Float32>(G(RandomishGenerator)).setRatio(1.1),
generateParam<Float64>(G(RandomishGenerator)).setRatio(1.1)
generateSeq<Int8 >(G(SequentialGenerator(1))),
generateSeq<Int16 >(G(SequentialGenerator(1))),
generateSeq<Int32 >(G(SequentialGenerator(1))),
generateSeq<Int64 >(G(SequentialGenerator(1))),
generateSeq<UInt8 >(G(SequentialGenerator(1))),
generateSeq<UInt16>(G(SequentialGenerator(1))),
generateSeq<UInt32>(G(SequentialGenerator(1))),
generateSeq<UInt64>(G(SequentialGenerator(1)))
)
),
);
INSTANTIATE_TEST_CASE_P(Overflow,
// -1, -2, -3, ... etc for signed
// 0xFF, 0xFE, 0xFD, ... for unsigned
INSTANTIATE_TEST_CASE_P(SequentialReverseInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateParam<UInt32>(G(MinMaxGenerator)),
generateParam<Int32>(G(MinMaxGenerator)),
generateParam<UInt64>(G(MinMaxGenerator)),
generateParam<Int64>(G(MinMaxGenerator))
generateSeq<Int8 >(G(SequentialGenerator(-1))),
generateSeq<Int16 >(G(SequentialGenerator(-1))),
generateSeq<Int32 >(G(SequentialGenerator(-1))),
generateSeq<Int64 >(G(SequentialGenerator(-1))),
generateSeq<UInt8 >(G(SequentialGenerator(-1))),
generateSeq<UInt16>(G(SequentialGenerator(-1))),
generateSeq<UInt32>(G(SequentialGenerator(-1))),
generateSeq<UInt64>(G(SequentialGenerator(-1)))
)
),
);
INSTANTIATE_TEST_CASE_P(SequentialFloat,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("Gorilla"),
Codec("Gorilla, LZ4")
),
::testing::Values(
generateSeq<Float32>(G(SequentialGenerator(M_E))),
generateSeq<Float64>(G(SequentialGenerator(M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(SequentialReverseFloat,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("Gorilla"),
Codec("Gorilla, LZ4")
),
::testing::Values(
generateSeq<Float32>(G(SequentialGenerator(-1 * M_E))),
generateSeq<Float64>(G(SequentialGenerator(-1 * M_E)))
)
),
);
INSTANTIATE_TEST_CASE_P(MonotonicInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateSeq<Int8 >(G(MonotonicGenerator(1, 5))),
generateSeq<Int16 >(G(MonotonicGenerator(1, 5))),
generateSeq<Int32 >(G(MonotonicGenerator(1, 5))),
generateSeq<Int64 >(G(MonotonicGenerator(1, 5))),
generateSeq<UInt8 >(G(MonotonicGenerator(1, 5))),
generateSeq<UInt16>(G(MonotonicGenerator(1, 5))),
generateSeq<UInt32>(G(MonotonicGenerator(1, 5))),
generateSeq<UInt64>(G(MonotonicGenerator(1, 5)))
)
),
);
INSTANTIATE_TEST_CASE_P(MonotonicReverseInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateSeq<Int8 >(G(MonotonicGenerator(-1, 5))),
generateSeq<Int16 >(G(MonotonicGenerator(-1, 5))),
generateSeq<Int32 >(G(MonotonicGenerator(-1, 5))),
generateSeq<Int64 >(G(MonotonicGenerator(-1, 5))),
generateSeq<UInt8 >(G(MonotonicGenerator(-1, 5))),
generateSeq<UInt16>(G(MonotonicGenerator(-1, 5))),
generateSeq<UInt32>(G(MonotonicGenerator(-1, 5))),
generateSeq<UInt64>(G(MonotonicGenerator(-1, 5)))
)
),
);
INSTANTIATE_TEST_CASE_P(MonotonicFloat,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("Gorilla")
),
::testing::Values(
generateSeq<Float32>(G(MonotonicGenerator<Float32>(M_E, 5))),
generateSeq<Float64>(G(MonotonicGenerator<Float64>(M_E, 5)))
)
),
);
INSTANTIATE_TEST_CASE_P(MonotonicReverseFloat,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("Gorilla")
),
::testing::Values(
generateSeq<Float32>(G(MonotonicGenerator<Float32>(-1 * M_E, 5))),
generateSeq<Float64>(G(MonotonicGenerator<Float64>(-1 * M_E, 5)))
)
),
);
INSTANTIATE_TEST_CASE_P(RandomInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateSeq<UInt8 >(G(RandomGenerator<UInt8>(0))),
generateSeq<UInt16>(G(RandomGenerator<UInt16>(0))),
generateSeq<UInt32>(G(RandomGenerator<UInt32>(0, 0, 1000'000'000))),
generateSeq<UInt64>(G(RandomGenerator<UInt64>(0, 0, 1000'000'000)))
)
),
);
INSTANTIATE_TEST_CASE_P(RandomishInt,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateSeq<Int32>(G(RandomishGenerator)),
generateSeq<Int64>(G(RandomishGenerator)),
generateSeq<UInt32>(G(RandomishGenerator)),
generateSeq<UInt64>(G(RandomishGenerator)),
generateSeq<Float32>(G(RandomishGenerator)),
generateSeq<Float64>(G(RandomishGenerator))
)
),
);
INSTANTIATE_TEST_CASE_P(RandomishFloat,
CodecTest,
::testing::Combine(
DefaultCodecsToTest,
::testing::Values(
generateSeq<Float32>(G(RandomishGenerator)),
generateSeq<Float64>(G(RandomishGenerator))
)
),
);
// Double delta overflow case, deltas are out of bounds for target type
INSTANTIATE_TEST_CASE_P(OverflowInt,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("DoubleDelta", 1.2),
Codec("DoubleDelta, LZ4", 1.0)
),
::testing::Values(
generateSeq<UInt32>(G(MinMaxGenerator())),
generateSeq<Int32>(G(MinMaxGenerator())),
generateSeq<UInt64>(G(MinMaxGenerator())),
generateSeq<Int64>(G(MinMaxGenerator()))
)
),
);
INSTANTIATE_TEST_CASE_P(OverflowFloat,
CodecTest,
::testing::Combine(
::testing::Values(
Codec("Gorilla", 1.1),
Codec("Gorilla, LZ4", 1.0)
),
::testing::Values(
generateSeq<Float32>(G(MinMaxGenerator())),
generateSeq<Float64>(G(MinMaxGenerator())),
generateSeq<Float32>(G(FFand0Generator())),
generateSeq<Float64>(G(FFand0Generator()))
)
),
);
}