From 923c6889e8bbe4aa5166e949c967dbdb5a493ea2 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 11 Aug 2019 11:01:02 +0300 Subject: [PATCH] Fixed Gorilla encoding error on small sequences. Added test cases for small sequences; Refurbished test cases for codecs; --- .../Compression/CompressionCodecGorilla.cpp | 20 +- dbms/src/Compression/ICompressionCodec.cpp | 2 +- .../tests/gtest_compressionCodec.cpp | 790 +++++++++++++----- 3 files changed, 592 insertions(+), 220 deletions(-) diff --git a/dbms/src/Compression/CompressionCodecGorilla.cpp b/dbms/src/Compression/CompressionCodecGorilla.cpp index 79cc6d27e81..8af6c8bfd39 100644 --- a/dbms/src/Compression/CompressionCodecGorilla.cpp +++ b/dbms/src/Compression/CompressionCodecGorilla.cpp @@ -78,11 +78,12 @@ binary_value_info getLeadingAndTrailingBits(const T & value) const UInt8 lz = getLeadingZeroBits(value); const UInt8 tz = getTrailingZeroBits(value); const UInt8 data_size = value == 0 ? 0 : static_cast(bit_size - lz - tz); + return binary_value_info{lz, data_size, tz}; } template -UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) +UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, UInt32 dest_size) { static const auto DATA_BIT_LENGTH = getBitLengthOfLength(sizeof(T)); // -1 since there must be at least 1 non-zero bit. @@ -91,6 +92,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) if (source_size % sizeof(T) != 0) throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; + const char * dest_end = dest + dest_size; const UInt32 items_count = source_size / sizeof(T); @@ -110,7 +112,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) dest += sizeof(prev_value); } - WriteBuffer buffer(dest, getCompressedDataSize(sizeof(T), source_size - sizeof(items_count) - sizeof(prev_value))); + WriteBuffer buffer(dest, dest_end - dest); BitWriter writer(buffer); while (source < source_end) @@ -265,24 +267,26 @@ UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 sourc dest[1] = bytes_to_skip; memcpy(&dest[2], source, bytes_to_skip); size_t start_pos = 2 + bytes_to_skip; - UInt32 compressed_size = 0; + UInt32 result_size = 0; + + const UInt32 compressed_size = getMaxCompressedDataSize(source_size); switch (data_bytes_size) { case 1: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + result_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size); break; case 2: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + result_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size); break; case 4: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + result_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size); break; case 8: - compressed_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos]); + result_size = compressDataForType(&source[bytes_to_skip], source_size - bytes_to_skip, &dest[start_pos], compressed_size); break; } - return 1 + 1 + compressed_size; + return 1 + 1 + result_size; } void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 /* uncompressed_size */) const diff --git a/dbms/src/Compression/ICompressionCodec.cpp b/dbms/src/Compression/ICompressionCodec.cpp index a50001238da..aafca2f5eb3 100644 --- a/dbms/src/Compression/ICompressionCodec.cpp +++ b/dbms/src/Compression/ICompressionCodec.cpp @@ -49,8 +49,8 @@ UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, ch UInt8 header_size = getHeaderSize(); UInt32 decompressed_size = unalignedLoad(&source[5]); doDecompressData(&source[header_size], source_size - header_size, dest, decompressed_size); - return decompressed_size; + return decompressed_size; } UInt32 ICompressionCodec::readCompressedBlockSize(const char * source) diff --git a/dbms/src/Compression/tests/gtest_compressionCodec.cpp b/dbms/src/Compression/tests/gtest_compressionCodec.cpp index 0f03070fff3..7fca3d98da7 100644 --- a/dbms/src/Compression/tests/gtest_compressionCodec.cpp +++ b/dbms/src/Compression/tests/gtest_compressionCodec.cpp @@ -1,10 +1,13 @@ -#include -#include +#include + #include #include #include #include +#include +#include +#include #include @@ -24,6 +27,33 @@ using namespace DB; +namespace std +{ +template +std::ostream & operator<<(std::ostream & ostr, const std::optional & opt) +{ + if (!opt) + { + return ostr << ""; + } + + return ostr << *opt; +} + +template +std::vector operator+(std::vector && left, std::vector && right) +{ + std::vector result(std::move(left)); + std::move(std::begin(right), std::end(right), std::back_inserter(result)); + + return result; +} + +} + +namespace +{ + template std::string bin(const T & value, size_t bits = sizeof(T)*8) { @@ -37,43 +67,46 @@ std::string bin(const T & value, size_t bits = sizeof(T)*8) template const char* type_name() { +#define MAKE_TYPE_NAME(TYPE) \ + if constexpr (std::is_same_v) return #TYPE; + + MAKE_TYPE_NAME(UInt8); + MAKE_TYPE_NAME(UInt16); + MAKE_TYPE_NAME(UInt32); + MAKE_TYPE_NAME(UInt64); + MAKE_TYPE_NAME(Int8); + MAKE_TYPE_NAME(Int16); + MAKE_TYPE_NAME(Int32); + MAKE_TYPE_NAME(Int64); + MAKE_TYPE_NAME(Float32); + MAKE_TYPE_NAME(Float64); + +#undef MAKE_TYPE_NAME + return typeid(T).name(); } -template <> -const char* type_name() +template +DataTypePtr makeDataType() { - return "uint32"; -} +#define MAKE_DATA_TYPE(TYPE) \ + if constexpr (std::is_same_v) return std::make_shared() -template <> -const char* type_name() -{ - return "int32"; -} + MAKE_DATA_TYPE(UInt8); + MAKE_DATA_TYPE(UInt16); + MAKE_DATA_TYPE(UInt32); + MAKE_DATA_TYPE(UInt64); + MAKE_DATA_TYPE(Int8); + MAKE_DATA_TYPE(Int16); + MAKE_DATA_TYPE(Int32); + MAKE_DATA_TYPE(Int64); + MAKE_DATA_TYPE(Float32); + MAKE_DATA_TYPE(Float64); -template <> -const char* type_name() -{ - return "uint64"; -} +#undef MAKE_DATA_TYPE -template <> -const char* type_name() -{ - return "int64"; -} - -template <> -const char* type_name() -{ - return "float"; -} - -template <> -const char* type_name() -{ - return "double"; + assert(false && "unsupported size"); + return nullptr; } @@ -135,52 +168,83 @@ template return result; } -struct CodecTestParam +struct CodecTestSequence { - std::string type_name; - std::vector source_data; - UInt8 data_byte_size; - double min_compression_ratio; - std::string case_name; + std::string name; + std::vector serialized_data; - // to allow setting ratio after building with complex builder functions. - CodecTestParam && setRatio(const double & ratio) && - { - this->min_compression_ratio = ratio; - return std::move(*this); - } + DataTypePtr data_type; }; -CodecTestParam operator+(CodecTestParam && left, CodecTestParam && right) +struct Codec { - assert(left.type_name == right.type_name); - assert(left.data_byte_size == right.data_byte_size); + std::string codec_statement; + std::optional expected_compression_ratio; - std::vector data(std::move(left.source_data)); - data.insert(data.end(), right.source_data.begin(), right.source_data.end()); + explicit Codec(std::string codec_statement_, std::optional expected_compression_ratio_ = std::nullopt) + : codec_statement(std::move(codec_statement_)), + expected_compression_ratio(expected_compression_ratio_) + {} - return CodecTestParam{ - left.type_name, - std::move(data), - left.data_byte_size, - std::min(left.min_compression_ratio, right.min_compression_ratio), - left.case_name + " + " + right.case_name + Codec() + : Codec(std::string()) + {} +}; + +CodecTestSequence operator+(CodecTestSequence && left, CodecTestSequence && right) +{ + assert(left.data_type->equals(*right.data_type)); + + std::vector data(std::move(left.serialized_data)); + data.insert(data.end(), right.serialized_data.begin(), right.serialized_data.end()); + + return CodecTestSequence{ + left.name + " + " + right.name, + std::move(data), + std::move(left.data_type) }; } -std::ostream & operator<<(std::ostream & ostr, const CodecTestParam & param) +template +CodecTestSequence operator*(CodecTestSequence && left, T times) { - return ostr << "name: " << param.case_name - << "\ntype name:" << param.type_name - << "\nbyte size: " << static_cast(param.data_byte_size) - << "\ndata size: " << param.source_data.size(); + std::vector data(std::move(left.serialized_data)); + const size_t initial_size = data.size(); + const size_t final_size = initial_size * times; + + data.reserve(final_size); + + for (T i = 0; i < times; ++i) + { + data.insert(data.end(), data.begin(), data.begin() + initial_size); + } + + return CodecTestSequence{ + left.name + " x " + std::to_string(times), + std::move(data), + std::move(left.data_type) + }; } -// compression ratio < 1.0 means that codec output is smaller than input. -const double DEFAULT_MIN_COMPRESSION_RATIO = 1.0; +std::ostream & operator<<(std::ostream & ostr, const Codec & codec) +{ + return ostr << "Codec{" + << "name: " << codec.codec_statement + << ", expected_compression_ratio: " << codec.expected_compression_ratio + << "}"; +} + +std::ostream & operator<<(std::ostream & ostr, const CodecTestSequence & seq) +{ + return ostr << "CodecTestSequence{" + << "name: " << seq.name + << ", type name: " << seq.data_type->getName() + << ", data size: " << seq.serialized_data.size() << " bytes" + << "}"; +} template -CodecTestParam makeParam(Args && ... args) +CodecTestSequence makeSeq(Args && ... args) { std::initializer_list vals{static_cast(args)...}; std::vector data(sizeof(T) * std::size(vals)); @@ -192,14 +256,17 @@ CodecTestParam makeParam(Args && ... args) write_pos += sizeof(v); } - return CodecTestParam{type_name(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO, - (boost::format("%1% values of %2%") % std::size(vals) % type_name()).str()}; + return CodecTestSequence{ + (boost::format("%1% values of %2%") % std::size(vals) % type_name()).str(), + std::move(data), + makeDataType() + }; } -template -CodecTestParam generateParam(Generator gen, const char* gen_name) +template +CodecTestSequence generateSeq(Generator gen, const char* gen_name, size_t Begin = 0, size_t End = 10000) { - static_assert (End >= Begin, "End must be not less than Begin"); + assert (End >= Begin); std::vector data(sizeof(T) * (End - Begin)); char * write_pos = data.data(); @@ -211,89 +278,104 @@ CodecTestParam generateParam(Generator gen, const char* gen_name) write_pos += sizeof(v); } - return CodecTestParam{type_name(), std::move(data), sizeof(T), DEFAULT_MIN_COMPRESSION_RATIO, - (boost::format("%1% values of %2% from %3%") % (End - Begin) % type_name() % gen_name).str()}; + return CodecTestSequence{ + (boost::format("%1% values of %2% from %3%") % (End - Begin) % type_name() % gen_name).str(), + std::move(data), + makeDataType() + }; } -void TestTranscoding(ICompressionCodec * codec, const CodecTestParam & param) -{ - const auto & source_data = param.source_data; - const UInt32 encoded_max_size = codec->getCompressedReserveSize(source_data.size()); - PODArray encoded(encoded_max_size); - - const UInt32 encoded_size = codec->compress(source_data.data(), source_data.size(), encoded.data()); - encoded.resize(encoded_size); - - PODArray decoded(source_data.size()); - const UInt32 decoded_size = codec->decompress(encoded.data(), encoded.size(), decoded.data()); - decoded.resize(decoded_size); - - switch (param.data_byte_size) - { - case 1: - ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); - break; - case 2: - ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); - break; - case 4: - ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); - break; - case 8: - ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); - break; - default: - FAIL() << "Invalid data_byte_size: " << param.data_byte_size; - } - const auto header_size = codec->getHeaderSize(); - const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0); - - ASSERT_LE(compression_ratio, param.min_compression_ratio) - << "\n\tdecoded size: " << source_data.size() - << "\n\tencoded size: " << encoded_size - << "(no header: " << encoded_size - header_size << ")"; -} - -class CodecTest : public ::testing::TestWithParam +class CodecTest : public ::testing::TestWithParam> { public: - static void SetUpTestCase() + enum MakeCodecParam { - // To make random predicatble and avoid failing test "out of the blue". - srand(0); + CODEC_WITH_DATA_TYPE, + CODEC_WITHOUT_DATA_TYPE, + }; + + CompressionCodecPtr makeCodec(MakeCodecParam with_data_type) const + { + const auto & codec_string = std::get<0>(GetParam()).codec_statement; + const auto & data_type = with_data_type == CODEC_WITH_DATA_TYPE ? std::get<1>(GetParam()).data_type : nullptr; + + const std::string codec_statement = "(" + codec_string + ")"; + Tokens tokens(codec_statement.begin().base(), codec_statement.end().base()); + TokenIterator token_iterator(tokens); + + Expected expected; + ASTPtr codec_ast; + ParserCodec parser; + parser.parse(token_iterator, codec_ast, expected); + + + return CompressionCodecFactory::instance().get(codec_ast, data_type); + } + + void testTranscoding(ICompressionCodec & codec) + { + const auto & test_sequence = std::get<1>(GetParam()); + const auto & source_data = test_sequence.serialized_data; + + const UInt32 encoded_max_size = codec.getCompressedReserveSize(source_data.size()); + PODArray encoded(encoded_max_size); + + const UInt32 encoded_size = codec.compress(source_data.data(), source_data.size(), encoded.data()); + encoded.resize(encoded_size); + + PODArray decoded(source_data.size()); + const UInt32 decoded_size = codec.decompress(encoded.data(), encoded.size(), decoded.data()); + decoded.resize(decoded_size); + + switch (test_sequence.data_type->getSizeOfValueInMemory()) + { + case 1: + ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); + break; + case 2: + ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); + break; + case 4: + ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); + break; + case 8: + ASSERT_TRUE(EqualByteContainersAs(source_data, decoded)); + break; + default: + FAIL() << "Invalid test sequence data type: " << test_sequence.data_type->getName(); + } + const auto header_size = codec.getHeaderSize(); + const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0); + + const auto & codec_spec = std::get<0>(GetParam()); + if (codec_spec.expected_compression_ratio) + { + ASSERT_LE(compression_ratio, *codec_spec.expected_compression_ratio) + << "\n\tdecoded size: " << source_data.size() + << "\n\tencoded size: " << encoded_size + << "(no header: " << encoded_size - header_size << ")"; + } } }; -TEST_P(CodecTest, DoubleDelta) +TEST_P(CodecTest, TranscodingWithDataType) { - auto param = GetParam(); - auto codec = std::make_unique(param.data_byte_size); - if (param.type_name == type_name() || param.type_name == type_name()) - { - // dd doesn't work great with many cases of integers and may result in very poor compression rate. - param.min_compression_ratio *= 1.5; - } - - TestTranscoding(codec.get(), param); + const auto codec = makeCodec(CODEC_WITH_DATA_TYPE); + testTranscoding(*codec); } -TEST_P(CodecTest, Gorilla) +TEST_P(CodecTest, TranscodingWithoutDataType) { - auto param = GetParam(); - auto codec = std::make_unique(param.data_byte_size); - if (param.type_name == type_name() || param.type_name == type_name() - || param.type_name == type_name() || param.type_name == type_name()) - { - // gorilla doesn't work great with many cases of integers and may result in very poor compression rate. - param.min_compression_ratio *= 1.5; - } - - TestTranscoding(codec.get(), param); + const auto codec = makeCodec(CODEC_WITHOUT_DATA_TYPE); + testTranscoding(*codec); } +/////////////////////////////////////////////////////////////////////////////////////////////////// // Here we use generators to produce test payload for codecs. -// Generator is a callable that should produce output value of the same type as input value. +// Generator is a callable that can produce infinite number of values, +// output value MUST be of the same type input value. +/////////////////////////////////////////////////////////////////////////////////////////////////// auto SameValueGenerator = [](auto value) { @@ -332,141 +414,427 @@ auto SequentialGenerator = [](auto stride = 1) //}; template +using uniform_distribution = +typename std::conditional_t, std::uniform_real_distribution, + typename std::conditional_t, std::uniform_int_distribution, void>>; + + +template struct MonotonicGenerator { - MonotonicGenerator(T stride_ = 1, size_t max_step_ = 10) + MonotonicGenerator(T stride_ = 1, T max_step = 10) : prev_value(0), stride(stride_), - max_step(max_step_) + random_engine(0), + distribution(0, max_step) {} template U operator()(U) { - const U result = prev_value + static_cast(stride * (rand() % max_step)); - - prev_value = result; - return result; + prev_value = prev_value + stride * distribution(random_engine); + return static_cast(prev_value); } +private: T prev_value; const T stride; - const size_t max_step; -}; - -auto MinMaxGenerator = [](auto i) -{ - if (i % 2 == 0) - { - return std::numeric_limits::min(); - } - else - { - return std::numeric_limits::max(); - } + std::default_random_engine random_engine; + uniform_distribution distribution; }; template struct RandomGenerator { - RandomGenerator(T seed = 0, T value_cap_ = std::numeric_limits::max()) - : e(seed), - value_cap(value_cap_) + RandomGenerator(T seed = 0, T value_min = std::numeric_limits::min(), T value_max = std::numeric_limits::max()) + : random_engine(seed), + distribution(value_min, value_max) { } template - U operator()(U i) + U operator()(U) { - return static_cast(distribution(e) % value_cap); + return static_cast(distribution(random_engine)); } private: - std::default_random_engine e; - std::uniform_int_distribution distribution; - const T value_cap; + std::default_random_engine random_engine; + uniform_distribution distribution; }; auto RandomishGenerator = [](auto i) { - return static_cast(sin(static_cast(i) * i) * i); + return static_cast(sin(static_cast(i * i)) * i); }; -// helper macro to produce human-friendly test case name +auto MinMaxGenerator = []() +{ + return [step = 0](auto i) mutable + { + if (step++ % 2 == 0) + { + return std::numeric_limits::min(); + } + else + { + return std::numeric_limits::max(); + } + }; +}; + +// Fill dest value with 0x00 or 0xFF +auto FFand0Generator = []() +{ + return [step = 0](auto i) mutable + { + decltype(i) result; + if (step++ % 2 == 0) + { + memset(&result, 0, sizeof(result)); + } + else + { + memset(&result, 0xFF, sizeof(result)); + } + + return result; + }; +}; + + +// Makes many sequences with generator, first sequence length is 1, second is 2... up to `sequences_count`. +template +std::vector generatePyramidOfSequences(const size_t sequences_count, Generator && generator, const char* generator_name) +{ + std::vector sequences; + sequences.reserve(sequences_count); + for (size_t i = 1; i < sequences_count; ++i) + { + std::string name = generator_name + std::string(" from 0 to ") + std::to_string(i); + sequences.push_back(generateSeq(std::forward(generator), name.c_str(), 0, i)); + } + + return sequences; +}; + + +// helper macro to produce human-friendly sequence name from generator #define G(generator) generator, #generator +const auto DefaultCodecsToTest = ::testing::Values( + Codec("DoubleDelta"), + Codec("DoubleDelta, LZ4"), + Codec("DoubleDelta, ZSTD"), + Codec("Gorilla"), + Codec("Gorilla, LZ4"), + Codec("Gorilla, ZSTD") +); + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// test cases +/////////////////////////////////////////////////////////////////////////////////////////////////// + +INSTANTIATE_TEST_CASE_P(Simple, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + makeSeq(1, 2, 3, 5, 7, 11, 13, 17, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(SmallSequences, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::ValuesIn( + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + + generatePyramidOfSequences(42, G(SequentialGenerator(1))) + ) + ), +); + INSTANTIATE_TEST_CASE_P(Mixed, CodecTest, - ::testing::Values( - generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), - generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), - generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1), - generateParam(G(MinMaxGenerator)) + generateParam(G(SequentialGenerator(1))).setRatio(1) + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001), + generateSeq(G(MinMaxGenerator()), 1, 5) + generateSeq(G(SequentialGenerator(1)), 1, 1001) + ) ), ); -INSTANTIATE_TEST_CASE_P(Same, +INSTANTIATE_TEST_CASE_P(SameValueInt, CodecTest, - ::testing::Values( - generateParam(G(SameValueGenerator(1000))), - generateParam(G(SameValueGenerator(-1000))), - generateParam(G(SameValueGenerator(1000))), - generateParam(G(SameValueGenerator(-1000))), - generateParam(G(SameValueGenerator(M_E))), - generateParam(G(SameValueGenerator(M_E))) + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))), + generateSeq(G(SameValueGenerator(1000))) + ) ), ); -INSTANTIATE_TEST_CASE_P(Sequential, +INSTANTIATE_TEST_CASE_P(SameNegativeValueInt, CodecTest, - ::testing::Values( - generateParam(G(SequentialGenerator(1))), - generateParam(G(SequentialGenerator(-1))), - generateParam(G(SequentialGenerator(1))), - generateParam(G(SequentialGenerator(-1))), - generateParam(G(SequentialGenerator(M_E))), - generateParam(G(SequentialGenerator(M_E))) + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))), + generateSeq(G(SameValueGenerator(-1000))) + ) ), ); -INSTANTIATE_TEST_CASE_P(Monotonic, +INSTANTIATE_TEST_CASE_P(SameValueFloat, CodecTest, - ::testing::Values( - generateParam(G(MonotonicGenerator(1, 5))), - generateParam(G(MonotonicGenerator(-1, 5))), - generateParam(G(MonotonicGenerator(1, 5))), - generateParam(G(MonotonicGenerator(-1, 5))), - generateParam(G(MonotonicGenerator(M_E, 5))), - generateParam(G(MonotonicGenerator(M_E, 5))) + ::testing::Combine( + ::testing::Values( + Codec("Gorilla"), + Codec("Gorilla, LZ4") + ), + ::testing::Values( + generateSeq(G(SameValueGenerator(M_E))), + generateSeq(G(SameValueGenerator(M_E))) + ) ), ); -INSTANTIATE_TEST_CASE_P(Random, +INSTANTIATE_TEST_CASE_P(SameNegativeValueFloat, CodecTest, - ::testing::Values( - generateParam(G(RandomGenerator(0, 1000'000'000))).setRatio(1.2), - generateParam(G(RandomGenerator(0, 1000'000'000))).setRatio(1.1) + ::testing::Combine( + ::testing::Values( + Codec("Gorilla"), + Codec("Gorilla, LZ4") + ), + ::testing::Values( + generateSeq(G(SameValueGenerator(-1 * M_E))), + generateSeq(G(SameValueGenerator(-1 * M_E))) + ) ), ); -INSTANTIATE_TEST_CASE_P(Randomish, +INSTANTIATE_TEST_CASE_P(SequentialInt, CodecTest, - ::testing::Values( - generateParam(G(RandomishGenerator)).setRatio(1.1), - generateParam(G(RandomishGenerator)).setRatio(1.1), - generateParam(G(RandomishGenerator)).setRatio(1.1), - generateParam(G(RandomishGenerator)).setRatio(1.1), - generateParam(G(RandomishGenerator)).setRatio(1.1), - generateParam(G(RandomishGenerator)).setRatio(1.1) + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))), + generateSeq(G(SequentialGenerator(1))) + ) ), ); -INSTANTIATE_TEST_CASE_P(Overflow, +// -1, -2, -3, ... etc for signed +// 0xFF, 0xFE, 0xFD, ... for unsigned +INSTANTIATE_TEST_CASE_P(SequentialReverseInt, CodecTest, - ::testing::Values( - generateParam(G(MinMaxGenerator)), - generateParam(G(MinMaxGenerator)), - generateParam(G(MinMaxGenerator)), - generateParam(G(MinMaxGenerator)) + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))), + generateSeq(G(SequentialGenerator(-1))) + ) ), ); + +INSTANTIATE_TEST_CASE_P(SequentialFloat, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("Gorilla"), + Codec("Gorilla, LZ4") + ), + ::testing::Values( + generateSeq(G(SequentialGenerator(M_E))), + generateSeq(G(SequentialGenerator(M_E))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(SequentialReverseFloat, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("Gorilla"), + Codec("Gorilla, LZ4") + ), + ::testing::Values( + generateSeq(G(SequentialGenerator(-1 * M_E))), + generateSeq(G(SequentialGenerator(-1 * M_E))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(MonotonicInt, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))), + generateSeq(G(MonotonicGenerator(1, 5))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(MonotonicReverseInt, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))), + generateSeq(G(MonotonicGenerator(-1, 5))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(MonotonicFloat, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("Gorilla") + ), + ::testing::Values( + generateSeq(G(MonotonicGenerator(M_E, 5))), + generateSeq(G(MonotonicGenerator(M_E, 5))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(MonotonicReverseFloat, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("Gorilla") + ), + ::testing::Values( + generateSeq(G(MonotonicGenerator(-1 * M_E, 5))), + generateSeq(G(MonotonicGenerator(-1 * M_E, 5))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(RandomInt, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(RandomGenerator(0))), + generateSeq(G(RandomGenerator(0))), + generateSeq(G(RandomGenerator(0, 0, 1000'000'000))), + generateSeq(G(RandomGenerator(0, 0, 1000'000'000))) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(RandomishInt, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(RandomishFloat, + CodecTest, + ::testing::Combine( + DefaultCodecsToTest, + ::testing::Values( + generateSeq(G(RandomishGenerator)), + generateSeq(G(RandomishGenerator)) + ) + ), +); + +// Double delta overflow case, deltas are out of bounds for target type +INSTANTIATE_TEST_CASE_P(OverflowInt, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("DoubleDelta", 1.2), + Codec("DoubleDelta, LZ4", 1.0) + ), + ::testing::Values( + generateSeq(G(MinMaxGenerator())), + generateSeq(G(MinMaxGenerator())), + generateSeq(G(MinMaxGenerator())), + generateSeq(G(MinMaxGenerator())) + ) + ), +); + +INSTANTIATE_TEST_CASE_P(OverflowFloat, + CodecTest, + ::testing::Combine( + ::testing::Values( + Codec("Gorilla", 1.1), + Codec("Gorilla, LZ4", 1.0) + ), + ::testing::Values( + generateSeq(G(MinMaxGenerator())), + generateSeq(G(MinMaxGenerator())), + generateSeq(G(FFand0Generator())), + generateSeq(G(FFand0Generator())) + ) + ), +); + +}