#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #pragma GCC diagnostic ignored "-Wmissing-declarations" #include using namespace DB; // Intentionally asymmetric both byte and word-size to detect read and write inconsistencies // each prime bit is set to 0. // v-61 v-53 v-47 v-41 v-37 v-31 v-23 v-17 v-11 v-5 const UInt64 BIT_PATTERN = 0b11101011'11101111'10111010'11101111'10101111'10111010'11101011'10101001; const uint8_t PRIMES[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61}; template std::string bin(const T & value, size_t bits = sizeof(T) * 8) { static const uint8_t MAX_BITS = sizeof(T)*8; assert(bits <= MAX_BITS); return std::bitset(static_cast(value)) .to_string().substr(MAX_BITS - bits, bits); } // gets N low bits of value template T getBits(uint8_t bits, const T & value) { return value & maskLowBits(bits); } template std::ostream & dumpBuffer(const T begin, const T end, std::ostream * destination, const char* col_sep = " ", const char* row_sep = "\n", const size_t cols_in_row = 8, UInt32 max_bytes = 0xFFFFFFFF) { size_t col = 0; for (auto p = begin; p < end && p - begin < max_bytes; ++p) { *destination << bin(*p); if (++col % cols_in_row == 0) { if (row_sep) *destination << row_sep; } else if (col_sep) { *destination << col_sep; } } return *destination; } template std::string dumpContents(const T& container, const char* col_sep = " ", const char* row_sep = "\n", const size_t cols_in_row = 8) { std::stringstream sstr; dumpBuffer(std::begin(container), std::end(container), &sstr, col_sep, row_sep, cols_in_row); return sstr.str(); } template ::testing::AssertionResult BinaryEqual(const ValueLeft & left, const ValueRight & right) { // ::testing::AssertionResult result = ::testing::AssertionSuccess(); if (sizeof(left) != sizeof(right)) return ::testing::AssertionFailure() << "Sizes do not match, expected: " << sizeof(left) << " actual: " << sizeof(right); const auto size = std::min(sizeof(left), sizeof(right)); if (memcmp(&left, &right, size) != 0) { const auto l_bits = left ? static_cast(std::log2(left)) : 0; const auto r_bits = right ? static_cast(std::log2(right)) : 0; const size_t bits = std::max(l_bits, r_bits) + 1; return ::testing::AssertionFailure() << "Values are binary different,\n" << "\texpected: 0b" << bin(left, bits) << " (" << std::hex << left << "),\n" << "\tactual : 0b" << bin(right, bits) << " (" <> bits_and_vals; std::string expected_buffer_binary; TestCaseParameter(std::vector> vals, std::string binary = std::string{}) // NOLINT : bits_and_vals(std::move(vals)), expected_buffer_binary(binary) {} }; class BitIO : public ::testing::TestWithParam {}; TEST_P(BitIO, WriteAndRead) { const auto & param = GetParam(); const auto & bits_and_vals = param.bits_and_vals; const auto & expected_buffer_binary = param.expected_buffer_binary; UInt64 max_buffer_size = 0; for (const auto & bv : bits_and_vals) { max_buffer_size += bv.first; } max_buffer_size = (max_buffer_size + 7) / 8; SCOPED_TRACE(max_buffer_size); PODArray data(max_buffer_size); { BitWriter writer(data.data(), data.size()); for (const auto & bv : bits_and_vals) { writer.writeBits(bv.first, bv.second); } writer.flush(); } { ReadBufferFromMemory read_buffer(data.data(), data.size()); // auto memory_read_buffer = memory_write_buffer.tryGetReadBuffer(); if (!expected_buffer_binary.empty()) { const auto actual_buffer_binary = dumpContents(data, " ", " "); ASSERT_EQ(expected_buffer_binary, actual_buffer_binary); } BitReader reader(data.data(), data.size()); int bitpos = 0; int item = 0; for (const auto & bv : bits_and_vals) { SCOPED_TRACE(::testing::Message() << "item #" << item << " of " << bits_and_vals.size() << ", width: " << static_cast(bv.first) << ", value: " << bv.second << "(" << bin(bv.second) << ")" << ", at bit position: " << std::dec << reader.count() << ".\nBuffer memory:\n" << dumpContents(data)); // const uint8_t next_byte = getBits(bv.first, bv.second) & ASSERT_TRUE(BinaryEqual(getBits(bv.first, bv.second), reader.readBits(bv.first))); ++item; bitpos += bv.first; } } } INSTANTIATE_TEST_SUITE_P(Simple, BitIO, ::testing::ValuesIn(std::initializer_list{ { {{9, 0xFFFFFFFF}, {9, 0x00}, {9, 0xFFFFFFFF}, {9, 0x00}, {9, 0xFFFFFFFF}}, "11111111 10000000 00111111 11100000 00001111 11111000 " }, { {{7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {3, 0xFFFF}}, "01111110 11111101 11111011 11110111 11101111 11011111 10111111 01111111 11000000 " }, { {{33, 0xFF110d0b07050300}, {33, 0xAAEE29251f1d1713}} }, { {{33, BIT_PATTERN}, {33, BIT_PATTERN}} }, { {{24, 0xFFFFFFFF}}, "11111111 11111111 11111111 " }, { // Note that we take only N lower bits of the number: {3, 0b01011} => 011 {{5, 0b01010}, {3, 0b111}, {7, 0b11001100}, {6, 0}, {5, 0b11111111}, {4, 0}, {3, 0b101}, {2, 0}, {1, 0b11111111}}, "01010111 10011000 00000111 11000010 10010000 " }, { {{64, BIT_PATTERN}, {56, BIT_PATTERN} , {4, 0b1111}, {4, 0}, // 128 {8, 0b11111111}, {64, BIT_PATTERN}, {48, BIT_PATTERN}, {8, 0}}, // 256 "11101011 11101111 10111010 11101111 10101111 10111010 11101011 10101001 " // 64 "11101111 10111010 11101111 10101111 10111010 11101011 10101001 11110000 " // 128 "11111111 11101011 11101111 10111010 11101111 10101111 10111010 11101011 " // 192 "10101001 10111010 11101111 10101111 10111010 11101011 10101001 00000000 " // 256 }, { {{64, BIT_PATTERN}, {56, BIT_PATTERN} , {5, 0b11111}, {3, 0}, // 128 {8, 0b11111111}, {64, BIT_PATTERN}, {48, BIT_PATTERN}, {8, 0}, //256 {32, BIT_PATTERN}, {12, 0xff}, {8, 0}, {12, 0xAEff}}, "11101011 11101111 10111010 11101111 10101111 10111010 11101011 10101001 " // 64 "11101111 10111010 11101111 10101111 10111010 11101011 10101001 11111000 " // 128 "11111111 11101011 11101111 10111010 11101111 10101111 10111010 11101011 " // 192 "10101001 10111010 11101111 10101111 10111010 11101011 10101001 00000000 " // 256 "10101111 10111010 11101011 10101001 00001111 11110000 00001110 11111111 " // 320 } }) ); TestCaseParameter primes_case(uint8_t repeat_times, UInt64 pattern) { std::vector> test_data; { for (uint8_t r = 0; r < repeat_times; ++r) { for (const auto p : PRIMES) { test_data.emplace_back(p, pattern); } } } return TestCaseParameter(test_data); } INSTANTIATE_TEST_SUITE_P(Primes, BitIO, ::testing::Values( primes_case(11, 0xFFFFFFFFFFFFFFFFULL), primes_case(11, BIT_PATTERN) ) ); TEST(BitHelpers, maskLowBits) { EXPECT_EQ(0b00000111, ::maskLowBits(3)); EXPECT_EQ(0b01111111, ::maskLowBits(7)); EXPECT_EQ(0b0000000001111111, ::maskLowBits(7)); EXPECT_EQ(0b0001111111111111, ::maskLowBits(13)); EXPECT_EQ(0b00000111111111111111111111111111, ::maskLowBits(27)); EXPECT_EQ(0b111111111111111111111111111111111, ::maskLowBits(33)); EXPECT_EQ(0b11111111111111111111111111111111111, ::maskLowBits(35)); EXPECT_EQ(0xFF, ::maskLowBits(8)); EXPECT_EQ(0xFFFF, ::maskLowBits(16)); EXPECT_EQ(0xFFFFFFFF, ::maskLowBits(32)); EXPECT_EQ(0xFFFFFFFFFFFFFFFF, ::maskLowBits(64)); }